LibSQL+SQLServer: Build SQLServer system service

This patch introduces the SQLServer system server. This service is
supposed to be the only process/application talking to database storage.
This makes things like locking and caching more reliable, easier to
implement, and more efficient.

In LibSQL we added a client component that does the ugly IPC nitty-
gritty for you. All that's needed is setting a number of event handler
lambdas and you can connect to databases and execute statements on them.

Applications that wish to use this SQLClient class obviously need to
link LibSQL and LibIPC.
This commit is contained in:
Jan de Visser 2021-06-28 21:15:17 -04:00 committed by Ali Mohammad Pur
parent 1037d6b0eb
commit a034774e3a
19 changed files with 650 additions and 12 deletions

View file

@ -386,6 +386,10 @@
#cmakedefine01 SQL_DEBUG
#endif
#ifndef SQLSERVER_DEBUG
#cmakedefine01 SQLSERVER_DEBUG
#endif
#ifndef SYNTAX_HIGHLIGHTING_DEBUG
#cmakedefine01 SYNTAX_HIGHLIGHTING_DEBUG
#endif

View file

@ -159,6 +159,7 @@ set(SOCKET_DEBUG ON)
set(SOLITAIRE_DEBUG ON)
set(SPAM_DEBUG ON)
set(SQL_DEBUG ON)
set(SQLSERVER_DEBUG ON)
set(STORAGE_DEVICE_DEBUG ON)
set(SYNTAX_HIGHLIGHTING_DEBUG ON)
set(SYSCALL_1_DEBUG ON)

View file

@ -1,5 +1,5 @@
file(GLOB TEST_SOURCES CONFIGURE_DEPENDS "*.cpp")
foreach(source ${TEST_SOURCES})
serenity_test(${source} LibSQL LIBS LibSQL)
serenity_test(${source} LibSQL LIBS LibSQL LibIPC)
endforeach()

View file

@ -14,10 +14,16 @@ set(SOURCES
Key.cpp
Meta.cpp
Row.cpp
SQLClient.cpp
TreeNode.cpp
Tuple.cpp
Value.cpp
)
set(GENERATED_SOURCES
../../Services/SQLServer/SQLClientEndpoint.h
../../Services/SQLServer/SQLServerEndpoint.h
)
serenity_lib(LibSQL sql)
target_link_libraries(LibSQL LibCore LibSyntax)

View file

@ -0,0 +1,75 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibSQL/SQLClient.h>
namespace SQL {
SQLClient::~SQLClient()
{
}
void SQLClient::connected(int connection_id)
{
if (on_connected)
on_connected(connection_id);
}
void SQLClient::disconnected(int connection_id)
{
if (on_disconnected)
on_disconnected(connection_id);
}
void SQLClient::connection_error(int connection_id, int code, String const& message)
{
if (on_connection_error)
on_connection_error(connection_id, code, message);
else
warnln("Connection error for connection_id {}: {} ({})", connection_id, message, code);
}
void SQLClient::execution_error(int statement_id, int code, String const& message)
{
if (on_execution_error)
on_execution_error(statement_id, code, message);
else
warnln("Execution error for statement_id {}: {} ({})", statement_id, message, code);
}
void SQLClient::execution_success(int statement_id, bool has_results, int created, int updated, int deleted)
{
if (on_execution_success)
on_execution_success(statement_id, has_results, created, updated, deleted);
else
outln("{} row(s) created, {} updated, {} deleted", created, updated, deleted);
}
void SQLClient::next_result(int statement_id, Vector<String> const& row)
{
if (on_next_result) {
on_next_result(statement_id, row);
return;
}
bool first = true;
for (auto& column : row) {
if (!first)
out(", ");
out("\"{}\"", column);
first = false;
}
outln();
}
void SQLClient::results_exhausted(int statement_id, int total_rows)
{
if (on_results_exhausted)
on_results_exhausted(statement_id, total_rows);
else
outln("{} total row(s)", total_rows);
}
}

View file

@ -0,0 +1,44 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibIPC/ServerConnection.h>
#include <SQLServer/SQLClientEndpoint.h>
#include <SQLServer/SQLServerEndpoint.h>
namespace SQL {
class SQLClient
: public IPC::ServerConnection<SQLClientEndpoint, SQLServerEndpoint>
, public SQLClientEndpoint {
C_OBJECT(SQLClient);
virtual ~SQLClient();
Function<void(int)> on_connected;
Function<void(int)> on_disconnected;
Function<void(int, int, String const&)> on_connection_error;
Function<void(int, int, String const&)> on_execution_error;
Function<void(int, bool, int, int, int)> on_execution_success;
Function<void(int, Vector<String> const&)> on_next_result;
Function<void(int, int)> on_results_exhausted;
private:
SQLClient()
: IPC::ServerConnection<SQLClientEndpoint, SQLServerEndpoint>(*this, "/tmp/portal/sql")
{
}
virtual void connected(int connection_id) override;
virtual void connection_error(int connection_id, int code, String const& message) override;
virtual void execution_success(int statement_id, bool has_results, int created, int updated, int deleted) override;
virtual void next_result(int statement_id, Vector<String> const&) override;
virtual void results_exhausted(int statement_id, int total_rows) override;
virtual void execution_error(int statement_id, int code, String const& message) override;
virtual void disconnected(int connection_id) override;
};
}

View file

@ -40,17 +40,18 @@ constexpr char const* command_tag(SQLCommand command)
}
}
#define ENUMERATE_SQL_ERRORS(S) \
S(NoError, "No error") \
S(DatabaseUnavailable, "Database Unavailable") \
S(StatementUnavailable, "Statement with id {} Unavailable") \
S(SyntaxError, "Syntax Error") \
S(DatabaseDoesNotExist, "Database {} does not exist") \
S(SchemaDoesNotExist, "Schema {} does not exist") \
S(SchemaExists, "Schema {} already exist") \
S(TableDoesNotExist, "Table {} does not exist") \
S(TableExists, "Table {} already exist") \
S(InvalidType, "Invalid type {}")
#define ENUMERATE_SQL_ERRORS(S) \
S(NoError, "No error") \
S(DatabaseUnavailable, "Database Unavailable") \
S(StatementUnavailable, "Statement with id '{}' Unavailable") \
S(SyntaxError, "Syntax Error") \
S(DatabaseDoesNotExist, "Database '{}' does not exist") \
S(SchemaDoesNotExist, "Schema '{}' does not exist") \
S(SchemaExists, "Schema '{}' already exist") \
S(TableDoesNotExist, "Table '{}' does not exist") \
S(TableExists, "Table '{}' already exist") \
S(InvalidType, "Invalid type '{}'") \
S(InvalidDatabaseName, "Invalid database name '{}'")
enum class SQLErrorCode {
#undef __ENUMERATE_SQL_ERROR
@ -120,6 +121,7 @@ private:
, m_update_count(update_count)
, m_insert_count(insert_count)
, m_delete_count(delete_count)
, m_has_results(command == SQLCommand::Select)
{
}

View file

@ -12,6 +12,7 @@ add_subdirectory(LaunchServer)
add_subdirectory(LookupServer)
add_subdirectory(NotificationServer)
add_subdirectory(RequestServer)
add_subdirectory(SQLServer)
add_subdirectory(SystemServer)
add_subdirectory(Taskbar)
add_subdirectory(TelnetServer)

View file

@ -0,0 +1,20 @@
serenity_component(
SQLServer
REQUIRED
TARGETS SQLServer
)
compile_ipc(SQLServer.ipc SQLServerEndpoint.h)
compile_ipc(SQLClient.ipc SQLClientEndpoint.h)
set(SOURCES
ClientConnection.cpp
DatabaseConnection.cpp
main.cpp
SQLClientEndpoint.h
SQLServerEndpoint.h
SQLStatement.cpp
)
serenity_bin(SQLServer)
target_link_libraries(SQLServer LibCore LibIPC LibSQL)

View file

@ -0,0 +1,84 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/String.h>
#include <AK/Vector.h>
#include <LibSQL/SQLResult.h>
#include <SQLServer/ClientConnection.h>
#include <SQLServer/DatabaseConnection.h>
#include <SQLServer/SQLStatement.h>
namespace SQLServer {
static HashMap<int, RefPtr<ClientConnection>> s_connections;
RefPtr<ClientConnection> ClientConnection::client_connection_for(int client_id)
{
if (s_connections.contains(client_id))
return *s_connections.get(client_id).value();
dbgln_if(SQLSERVER_DEBUG, "Invalid client_id {}", client_id);
return nullptr;
}
ClientConnection::ClientConnection(AK::NonnullRefPtr<Core::LocalSocket> socket, int client_id)
: IPC::ClientConnection<SQLClientEndpoint, SQLServerEndpoint>(*this, move(socket), client_id)
{
s_connections.set(client_id, *this);
}
ClientConnection::~ClientConnection()
{
}
void ClientConnection::die()
{
s_connections.remove(client_id());
}
Messages::SQLServer::ConnectResponse ClientConnection::connect(String const& database_name)
{
dbgln_if(SQLSERVER_DEBUG, "ClientConnection::connect(database_name: {})", database_name);
auto database_connection = DatabaseConnection::construct(database_name, client_id());
return { database_connection->connection_id() };
}
void ClientConnection::disconnect(int connection_id)
{
dbgln_if(SQLSERVER_DEBUG, "ClientConnection::disconnect(connection_id: {})", connection_id);
auto database_connection = DatabaseConnection::connection_for(connection_id);
if (database_connection)
database_connection->disconnect();
else
dbgln("Database connection has disappeared");
}
Messages::SQLServer::SqlStatementResponse ClientConnection::sql_statement(int connection_id, String const& sql)
{
dbgln_if(SQLSERVER_DEBUG, "ClientConnection::sql_statement(connection_id: {}, sql: '{}')", connection_id, sql);
auto database_connection = DatabaseConnection::connection_for(connection_id);
if (database_connection) {
auto statement_id = database_connection->sql_statement(sql);
dbgln_if(SQLSERVER_DEBUG, "ClientConnection::sql_statement -> statement_id = {}", statement_id);
return { statement_id };
} else {
dbgln("Database connection has disappeared");
return { -1 };
}
}
void ClientConnection::statement_execute(int statement_id)
{
dbgln_if(SQLSERVER_DEBUG, "ClientConnection::statement_execute_query(statement_id: {})", statement_id);
auto statement = SQLStatement::statement_for(statement_id);
if (statement && statement->connection()->client_id() == client_id()) {
statement->execute();
} else {
dbgln_if(SQLSERVER_DEBUG, "Statement has disappeared");
async_execution_error(statement_id, (int)SQL::SQLErrorCode::StatementUnavailable, String::formatted("{}", statement_id));
}
}
}

View file

@ -0,0 +1,35 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/HashMap.h>
#include <LibIPC/ClientConnection.h>
#include <SQLServer/SQLClientEndpoint.h>
#include <SQLServer/SQLServerEndpoint.h>
namespace SQLServer {
class ClientConnection final
: public IPC::ClientConnection<SQLClientEndpoint, SQLServerEndpoint> {
C_OBJECT(ClientConnection);
public:
explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
virtual ~ClientConnection() override;
virtual void die() override;
static RefPtr<ClientConnection> client_connection_for(int client_id);
private:
virtual Messages::SQLServer::ConnectResponse connect(String const&) override;
virtual Messages::SQLServer::SqlStatementResponse sql_statement(int, String const&) override;
virtual void statement_execute(int) override;
virtual void disconnect(int) override;
};
}

View file

@ -0,0 +1,83 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/LexicalPath.h>
#include <SQLServer/ClientConnection.h>
#include <SQLServer/DatabaseConnection.h>
#include <SQLServer/SQLStatement.h>
namespace SQLServer {
static HashMap<int, NonnullRefPtr<DatabaseConnection>> s_connections;
RefPtr<DatabaseConnection> DatabaseConnection::connection_for(int connection_id)
{
if (s_connections.contains(connection_id))
return *s_connections.get(connection_id).value();
dbgln_if(SQLSERVER_DEBUG, "Invalid connection_id {}", connection_id);
return nullptr;
}
static int s_next_connection_id = 0;
DatabaseConnection::DatabaseConnection(String database_name, int client_id)
: Object()
, m_database_name(move(database_name))
, m_connection_id(s_next_connection_id++)
, m_client_id(client_id)
{
LexicalPath path(database_name);
if (path.title() != database_name) {
auto client_connection = ClientConnection::client_connection_for(m_client_id);
client_connection->async_connection_error(m_connection_id, (int)SQL::SQLErrorCode::InvalidDatabaseName, m_database_name);
return;
}
dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection {} initiating connection with database '{}'", connection_id(), m_database_name);
s_connections.set(m_connection_id, *this);
deferred_invoke([&](Object&) {
m_database = SQL::Database::construct(String::formatted("/home/anon/sql/{}.db", m_database_name));
m_accept_statements = true;
auto client_connection = ClientConnection::client_connection_for(client_id);
if (client_connection)
client_connection->async_connected(m_connection_id);
else
warnln("Cannot notify client of database connection. Client disconnected");
});
}
void DatabaseConnection::disconnect()
{
dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection::disconnect(connection_id {}, database '{}'", connection_id(), m_database_name);
m_accept_statements = false;
deferred_invoke([&](Object&) {
m_database = nullptr;
s_connections.remove(m_connection_id);
auto client_connection = ClientConnection::client_connection_for(client_id());
if (client_connection)
client_connection->async_disconnected(m_connection_id);
else
warnln("Cannot notify client of database disconnection. Client disconnected");
});
}
int DatabaseConnection::sql_statement(String const& sql)
{
dbgln_if(SQLSERVER_DEBUG, "DatabaseConnection::sql_statement(connection_id {}, database '{}', sql '{}'", connection_id(), m_database_name, sql);
auto client_connection = ClientConnection::client_connection_for(client_id());
if (!client_connection) {
warnln("Cannot notify client of database disconnection. Client disconnected");
return -1;
}
if (!m_accept_statements) {
client_connection->async_execution_error(-1, (int)SQL::SQLErrorCode::DatabaseUnavailable, m_database_name);
return -1;
}
auto statement = SQLStatement::construct(*this, sql);
return statement->statement_id();
}
}

View file

@ -0,0 +1,36 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibCore/Object.h>
#include <LibSQL/Database.h>
#include <SQLServer/Forward.h>
namespace SQLServer {
class DatabaseConnection final : public Core::Object {
C_OBJECT(DatabaseConnection)
~DatabaseConnection() override = default;
static RefPtr<DatabaseConnection> connection_for(int connection_id);
int connection_id() const { return m_connection_id; }
int client_id() const { return m_client_id; }
RefPtr<SQL::Database> database() { return m_database; }
void disconnect();
int sql_statement(String const& sql);
private:
DatabaseConnection(String database_name, int client_id);
RefPtr<SQL::Database> m_database { nullptr };
String m_database_name;
int m_connection_id;
int m_client_id;
bool m_accept_statements { false };
};
}

View file

@ -0,0 +1,13 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
namespace SQLServer {
class ClientConnection;
class DatabaseConnection;
class SQLStatement;
}

View file

@ -0,0 +1,10 @@
endpoint SQLClient
{
connected(int connection_id) =|
connection_error(int connection_id, int code, String message) =|
execution_success(int statement_id, bool has_results, int created, int updated, int deleted) =|
next_result(int statement_id, Vector<String> row) =|
results_exhausted(int statement_id, int total_rows) =|
execution_error(int statement_id, int code, String message) =|
disconnected(int connection_id) =|
}

View file

@ -0,0 +1,7 @@
endpoint SQLServer [magic=5432]
{
connect(String name) => (int connection_id)
sql_statement(int connection_id, String statement) => (int statement_id)
statement_execute(int statement_id) =|
disconnect(int connection_id) =|
}

View file

@ -0,0 +1,116 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibCore/Object.h>
#include <LibSQL/AST/Parser.h>
#include <SQLServer/ClientConnection.h>
#include <SQLServer/DatabaseConnection.h>
#include <SQLServer/SQLStatement.h>
namespace SQLServer {
static HashMap<int, NonnullRefPtr<SQLStatement>> s_statements;
RefPtr<SQLStatement> SQLStatement::statement_for(int statement_id)
{
if (s_statements.contains(statement_id))
return *s_statements.get(statement_id).value();
dbgln_if(SQLSERVER_DEBUG, "Invalid statement_id {}", statement_id);
return nullptr;
}
static int s_next_statement_id = 0;
SQLStatement::SQLStatement(DatabaseConnection& connection, String sql)
: Core::Object(&connection)
, m_statement_id(s_next_statement_id++)
, m_sql(move(sql))
{
dbgln_if(SQLSERVER_DEBUG, "SQLStatement({}, {})", connection.connection_id(), sql);
s_statements.set(m_statement_id, *this);
}
void SQLStatement::report_error(SQL::SQLError error)
{
dbgln_if(SQLSERVER_DEBUG, "SQLStatement::report_error(statement_id {}, error {}", statement_id(), error.to_string());
auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
m_statement = nullptr;
m_result = nullptr;
remove_from_parent();
s_statements.remove(statement_id());
if (!client_connection) {
warnln("Cannot return execution error. Client disconnected");
warnln("SQLStatement::report_error(statement_id {}, error {}", statement_id(), error.to_string());
m_result = nullptr;
return;
}
client_connection->async_execution_error(statement_id(), (int)error.code, error.to_string());
m_result = nullptr;
}
void SQLStatement::execute()
{
dbgln_if(SQLSERVER_DEBUG, "SQLStatement::execute(statement_id {}", statement_id());
auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
if (!client_connection) {
warnln("Cannot yield next result. Client disconnected");
return;
}
deferred_invoke([&](Object&) {
auto maybe_error = parse();
if (maybe_error.has_value()) {
report_error(maybe_error.value());
return;
}
m_result = m_statement->execute(*connection()->database());
if (m_result->error().code != SQL::SQLErrorCode::NoError) {
report_error(m_result->error());
return;
}
auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
if (!client_connection) {
warnln("Cannot return statement execution results. Client disconnected");
return;
}
client_connection->async_execution_success(statement_id(), m_result->has_results(), m_result->updated(), m_result->inserted(), m_result->deleted());
if (m_result->has_results()) {
m_index = 0;
next();
}
});
}
Optional<SQL::SQLError> SQLStatement::parse()
{
auto parser = SQL::AST::Parser(SQL::AST::Lexer(m_sql));
m_statement = parser.next_statement();
if (parser.has_errors()) {
return SQL::SQLError { SQL::SQLErrorCode::SyntaxError, parser.errors()[0].to_string() };
}
return {};
}
void SQLStatement::next()
{
VERIFY(m_result->has_results());
auto client_connection = ClientConnection::client_connection_for(connection()->client_id());
if (!client_connection) {
warnln("Cannot yield next result. Client disconnected");
return;
}
if (m_index < m_result->results().size()) {
auto& tuple = m_result->results()[m_index++];
client_connection->async_next_result(statement_id(), tuple.to_string_vector());
deferred_invoke([&](Object&) {
next();
});
} else {
client_connection->async_results_exhausted(statement_id(), (int)m_index);
}
}
}

View file

@ -0,0 +1,42 @@
/*
* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/NonnullRefPtr.h>
#include <AK/String.h>
#include <LibCore/Object.h>
#include <LibSQL/AST/AST.h>
#include <LibSQL/SQLResult.h>
#include <SQLServer/DatabaseConnection.h>
#include <SQLServer/Forward.h>
namespace SQLServer {
class SQLStatement final : public Core::Object {
C_OBJECT(SQLStatement)
~SQLStatement() override = default;
static RefPtr<SQLStatement> statement_for(int statement_id);
int statement_id() const { return m_statement_id; }
String const& sql() const { return m_sql; }
DatabaseConnection* connection() { return dynamic_cast<DatabaseConnection*>(parent()); }
void execute();
private:
SQLStatement(DatabaseConnection&, String sql);
Optional<SQL::SQLError> parse();
void next();
void report_error(SQL::SQLError);
int m_statement_id;
String m_sql;
size_t m_index { 0 };
RefPtr<SQL::AST::Statement> m_statement { nullptr };
RefPtr<SQL::SQLResult> m_result { nullptr };
};
}

View file

@ -0,0 +1,59 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibCore/EventLoop.h>
#include <LibCore/LocalServer.h>
#include <SQLServer/ClientConnection.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv)
{
if (pledge("stdio accept unix rpath wpath cpath", nullptr) < 0) {
perror("pledge");
return 1;
}
struct stat statbuf;
if (stat("/home/anon/sql", &statbuf) != 0) {
if (errno != ENOENT) {
perror("stat");
return 1;
}
if (mkdir("/home/anon/sql", 0700) != 0) {
perror("mkdir");
return 1;
}
}
if (unveil("/home/anon/sql", "rwc") < 0) {
perror("unveil");
return 1;
}
if (unveil(nullptr, nullptr) < 0) {
perror("unveil");
return 1;
}
Core::EventLoop event_loop;
auto server = Core::LocalServer::construct();
bool ok = server->take_over_from_system_server();
VERIFY(ok);
server->on_ready_to_accept = [&] {
auto client_socket = server->accept();
if (!client_socket) {
dbgln("SQLServer: accept failed.");
return;
}
static int s_next_client_id = 0;
int client_id = ++s_next_client_id;
IPC::new_client_connection<SQLServer::ClientConnection>(client_socket.release_nonnull(), client_id);
};
return event_loop.exec();
}