LibWebView+WebContent: Use Web::InputEvent for WebContent input IPC

Now that all input events are handled by LibWebView, replace the IPCs
which send the fields of Web::KeyEvent / Web::MouseEvent individually
with one IPC per event type (key or mouse).

We can also replace the ad-hoc queued input structure with a smaller
struct that simply holds the tranferred Web::KeyEvent / Web::MouseEvent.

In the future, we can also adapt Web::EventHandler to use these structs.
This commit is contained in:
Timothy Flynn 2024-03-05 17:06:32 -05:00 committed by Andreas Kling
parent 2c31ef11bc
commit baf359354b
8 changed files with 164 additions and 228 deletions

View file

@ -7,6 +7,7 @@ source_set("Page") {
sources = [
"EditEventHandler.cpp",
"EventHandler.cpp",
"InputEvent.cpp",
"Page.cpp",
]
}

View file

@ -492,6 +492,7 @@ set(SOURCES
NavigationTiming/PerformanceTiming.cpp
Page/EditEventHandler.cpp
Page/EventHandler.cpp
Page/InputEvent.cpp
Page/Page.cpp
Painting/AudioPaintable.cpp
Painting/BackgroundPainting.cpp

View file

@ -0,0 +1,73 @@
/*
* Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibIPC/Decoder.h>
#include <LibIPC/Encoder.h>
#include <LibWeb/Page/InputEvent.h>
namespace Web {
KeyEvent KeyEvent::clone_without_chrome_data() const
{
return { type, key, modifiers, code_point, nullptr };
}
MouseEvent MouseEvent::clone_without_chrome_data() const
{
return { type, position, screen_position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y, nullptr };
}
}
template<>
ErrorOr<void> IPC::encode(Encoder& encoder, Web::KeyEvent const& event)
{
TRY(encoder.encode(event.type));
TRY(encoder.encode(event.key));
TRY(encoder.encode(event.modifiers));
TRY(encoder.encode(event.code_point));
return {};
}
template<>
ErrorOr<Web::KeyEvent> IPC::decode(Decoder& decoder)
{
auto type = TRY(decoder.decode<Web::KeyEvent::Type>());
auto key = TRY(decoder.decode<KeyCode>());
auto modifiers = TRY(decoder.decode<KeyModifier>());
auto code_point = TRY(decoder.decode<u32>());
return Web::KeyEvent { type, key, modifiers, code_point, nullptr };
}
template<>
ErrorOr<void> IPC::encode(Encoder& encoder, Web::MouseEvent const& event)
{
TRY(encoder.encode(event.type));
TRY(encoder.encode(event.position));
TRY(encoder.encode(event.screen_position));
TRY(encoder.encode(event.button));
TRY(encoder.encode(event.buttons));
TRY(encoder.encode(event.modifiers));
TRY(encoder.encode(event.wheel_delta_x));
TRY(encoder.encode(event.wheel_delta_y));
return {};
}
template<>
ErrorOr<Web::MouseEvent> IPC::decode(Decoder& decoder)
{
auto type = TRY(decoder.decode<Web::MouseEvent::Type>());
auto position = TRY(decoder.decode<Web::DevicePixelPoint>());
auto screen_position = TRY(decoder.decode<Web::DevicePixelPoint>());
auto button = TRY(decoder.decode<GUI::MouseButton>());
auto buttons = TRY(decoder.decode<GUI::MouseButton>());
auto modifiers = TRY(decoder.decode<KeyModifier>());
auto wheel_delta_x = TRY(decoder.decode<int>());
auto wheel_delta_y = TRY(decoder.decode<int>());
return Web::MouseEvent { type, position, screen_position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y, nullptr };
}

View file

@ -9,6 +9,7 @@
#include <AK/OwnPtr.h>
#include <AK/Variant.h>
#include <LibGfx/Point.h>
#include <LibIPC/Forward.h>
#include <LibWeb/PixelUnits.h>
// FIXME: These should not be included outside of Serenity. This FIXME appears in several locations across the Ladybird
@ -29,6 +30,8 @@ public:
KeyUp,
};
KeyEvent clone_without_chrome_data() const;
Type type;
KeyCode key { KeyCode::Key_Invalid };
KeyModifier modifiers { KeyModifier::Mod_None };
@ -47,6 +50,8 @@ public:
DoubleClick,
};
MouseEvent clone_without_chrome_data() const;
Type type;
Web::DevicePixelPoint position;
Web::DevicePixelPoint screen_position;
@ -62,3 +67,19 @@ public:
using InputEvent = Variant<KeyEvent, MouseEvent>;
}
namespace IPC {
template<>
ErrorOr<void> encode(Encoder&, Web::KeyEvent const&);
template<>
ErrorOr<Web::KeyEvent> decode(Decoder&);
template<>
ErrorOr<void> encode(Encoder&, Web::MouseEvent const&);
template<>
ErrorOr<Web::MouseEvent> decode(Decoder&);
}

View file

@ -119,36 +119,12 @@ void ViewImplementation::enqueue_input_event(Web::InputEvent event)
// process the next one.
m_pending_input_events.enqueue(move(event));
// FIXME: Replace these IPCs with a singular "async_input_event".
m_pending_input_events.tail().visit(
[this](Web::KeyEvent const& event) {
switch (event.type) {
case Web::KeyEvent::Type::KeyDown:
client().async_key_down(m_client_state.page_index, event.key, event.modifiers, event.code_point);
break;
case Web::KeyEvent::Type::KeyUp:
client().async_key_up(m_client_state.page_index, event.key, event.modifiers, event.code_point);
break;
}
client().async_key_event(m_client_state.page_index, event.clone_without_chrome_data());
},
[this](Web::MouseEvent const& event) {
switch (event.type) {
case Web::MouseEvent::Type::MouseDown:
client().async_mouse_down(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
break;
case Web::MouseEvent::Type::MouseUp:
client().async_mouse_up(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
break;
case Web::MouseEvent::Type::MouseMove:
client().async_mouse_move(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
break;
case Web::MouseEvent::Type::MouseWheel:
client().async_mouse_wheel(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y);
break;
case Web::MouseEvent::Type::DoubleClick:
client().async_doubleclick(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
break;
}
client().async_mouse_event(m_client_state.page_index, event.clone_without_chrome_data());
});
}

View file

@ -227,185 +227,84 @@ void ConnectionFromClient::process_next_input_event()
return;
auto event = m_input_event_queue.dequeue();
event.visit(
[&](QueuedMouseEvent const& event) {
auto maybe_page = page(event.page_id);
if (!maybe_page.has_value()) {
dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
return;
}
auto& page = maybe_page.release_value();
auto maybe_page = page(event.page_id);
if (!maybe_page.has_value()) {
dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
return;
}
auto& page = maybe_page->page();
auto handled = event.event.visit(
[&](Web::KeyEvent const& event) {
switch (event.type) {
case QueuedMouseEvent::Type::MouseDown:
report_finished_handling_input_event(event.page_id, page.page().handle_mousedown(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
break;
case QueuedMouseEvent::Type::MouseUp:
report_finished_handling_input_event(event.page_id, page.page().handle_mouseup(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
break;
case QueuedMouseEvent::Type::MouseMove:
// NOTE: We have to notify the client about coalesced MouseMoves,
// so we do that by saying none of them were handled by the web page.
for (size_t i = 0; i < event.coalesced_event_count; ++i) {
report_finished_handling_input_event(event.page_id, false);
}
report_finished_handling_input_event(event.page_id, page.page().handle_mousemove(event.position, event.screen_position, event.buttons, event.modifiers));
break;
case QueuedMouseEvent::Type::DoubleClick:
report_finished_handling_input_event(event.page_id, page.page().handle_doubleclick(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
break;
case QueuedMouseEvent::Type::MouseWheel:
for (size_t i = 0; i < event.coalesced_event_count; ++i) {
report_finished_handling_input_event(event.page_id, false);
}
report_finished_handling_input_event(event.page_id, page.page().handle_mousewheel(event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y));
break;
case Web::KeyEvent::Type::KeyDown:
return page.handle_keydown(event.key, event.modifiers, event.code_point);
case Web::KeyEvent::Type::KeyUp:
return page.handle_keyup(event.key, event.modifiers, event.code_point);
}
VERIFY_NOT_REACHED();
},
[&](QueuedKeyboardEvent const& event) {
auto maybe_page = page(event.page_id);
if (!maybe_page.has_value()) {
dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
return;
}
auto& page = maybe_page.release_value();
[&](Web::MouseEvent const& event) {
switch (event.type) {
case QueuedKeyboardEvent::Type::KeyDown:
report_finished_handling_input_event(event.page_id, page.page().handle_keydown((KeyCode)event.key, event.modifiers, event.code_point));
break;
case QueuedKeyboardEvent::Type::KeyUp:
report_finished_handling_input_event(event.page_id, page.page().handle_keyup((KeyCode)event.key, event.modifiers, event.code_point));
break;
case Web::MouseEvent::Type::MouseDown:
return page.handle_mousedown(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
case Web::MouseEvent::Type::MouseUp:
return page.handle_mouseup(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
case Web::MouseEvent::Type::MouseMove:
return page.handle_mousemove(event.position, event.screen_position, event.buttons, event.modifiers);
case Web::MouseEvent::Type::MouseWheel:
return page.handle_mousewheel(event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y);
case Web::MouseEvent::Type::DoubleClick:
return page.handle_doubleclick(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
}
VERIFY_NOT_REACHED();
});
// We have to notify the client about coalesced events, so we do that by saying none of them were handled by the web page.
for (size_t i = 0; i < event.coalesced_event_count; ++i)
report_finished_handling_input_event(event.page_id, false);
report_finished_handling_input_event(event.page_id, handled);
if (!m_input_event_queue.is_empty())
m_input_event_queue_timer->start();
}
void ConnectionFromClient::mouse_down(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
void ConnectionFromClient::key_event(u64 page_id, Web::KeyEvent const& event)
{
enqueue_input_event(
QueuedMouseEvent {
.type = QueuedMouseEvent::Type::MouseDown,
.position = position,
.screen_position = screen_position,
.button = button,
.buttons = buttons,
.modifiers = modifiers,
.page_id = page_id,
});
enqueue_input_event({ page_id, move(const_cast<Web::KeyEvent&>(event)), 0 });
}
void ConnectionFromClient::mouse_move(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, [[maybe_unused]] u32 button, u32 buttons, u32 modifiers)
void ConnectionFromClient::mouse_event(u64 page_id, Web::MouseEvent const& event)
{
auto event = QueuedMouseEvent {
.type = QueuedMouseEvent::Type::MouseMove,
.position = position,
.screen_position = screen_position,
.button = button,
.buttons = buttons,
.modifiers = modifiers,
.page_id = page_id,
// OPTIMIZATION: Coalesce consecutive unprocessed mouse move and wheel events.
auto should_coalesce = [&]() {
if (m_input_event_queue.is_empty())
return false;
if (event.type != Web::MouseEvent::Type::MouseMove && event.type != Web::MouseEvent::Type::MouseWheel)
return false;
if (m_input_event_queue.tail().page_id != page_id)
return false;
if (auto* mouse_event = m_input_event_queue.tail().event.get_pointer<Web::MouseEvent>())
return mouse_event->type == event.type;
return false;
};
// OPTIMIZATION: Coalesce with previous unprocessed event iff the previous event is also a MouseMove event.
if (!m_input_event_queue.is_empty()
&& m_input_event_queue.tail().has<QueuedMouseEvent>()
&& m_input_event_queue.tail().get<QueuedMouseEvent>().type == QueuedMouseEvent::Type::MouseMove
&& m_input_event_queue.tail().get<QueuedMouseEvent>().page_id == page_id) {
event.coalesced_event_count = m_input_event_queue.tail().get<QueuedMouseEvent>().coalesced_event_count + 1;
m_input_event_queue.tail() = event;
if (should_coalesce()) {
m_input_event_queue.tail().event = move(const_cast<Web::MouseEvent&>(event));
++m_input_event_queue.tail().coalesced_event_count;
return;
}
enqueue_input_event(move(event));
enqueue_input_event({ page_id, move(const_cast<Web::MouseEvent&>(event)), 0 });
}
void ConnectionFromClient::mouse_up(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
{
enqueue_input_event(
QueuedMouseEvent {
.type = QueuedMouseEvent::Type::MouseUp,
.position = position,
.screen_position = screen_position,
.button = button,
.buttons = buttons,
.modifiers = modifiers,
.page_id = page_id,
});
}
void ConnectionFromClient::mouse_wheel(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers, Web::DevicePixels wheel_delta_x, Web::DevicePixels wheel_delta_y)
{
auto event = QueuedMouseEvent {
.type = QueuedMouseEvent::Type::MouseWheel,
.position = position,
.screen_position = screen_position,
.button = button,
.buttons = buttons,
.modifiers = modifiers,
.wheel_delta_x = wheel_delta_x,
.wheel_delta_y = wheel_delta_y,
.page_id = page_id,
};
// OPTIMIZATION: Coalesce with previous unprocessed event if the previous event is also a MouseWheel event.
if (!m_input_event_queue.is_empty()
&& m_input_event_queue.tail().has<QueuedMouseEvent>()
&& m_input_event_queue.tail().get<QueuedMouseEvent>().type == QueuedMouseEvent::Type::MouseWheel
&& m_input_event_queue.tail().get<QueuedMouseEvent>().page_id == page_id) {
auto const& last_event = m_input_event_queue.tail().get<QueuedMouseEvent>();
event.coalesced_event_count = last_event.coalesced_event_count + 1;
event.wheel_delta_x += last_event.wheel_delta_x;
event.wheel_delta_y += last_event.wheel_delta_y;
m_input_event_queue.tail() = event;
return;
}
enqueue_input_event(move(event));
}
void ConnectionFromClient::doubleclick(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
{
enqueue_input_event(
QueuedMouseEvent {
.type = QueuedMouseEvent::Type::DoubleClick,
.position = position,
.screen_position = screen_position,
.button = button,
.buttons = buttons,
.modifiers = modifiers,
.page_id = page_id,
});
}
void ConnectionFromClient::key_down(u64 page_id, i32 key, u32 modifiers, u32 code_point)
{
enqueue_input_event(
QueuedKeyboardEvent {
.type = QueuedKeyboardEvent::Type::KeyDown,
.key = key,
.modifiers = modifiers,
.code_point = code_point,
.page_id = page_id,
});
}
void ConnectionFromClient::key_up(u64 page_id, i32 key, u32 modifiers, u32 code_point)
{
enqueue_input_event(
QueuedKeyboardEvent {
.type = QueuedKeyboardEvent::Type::KeyUp,
.key = key,
.modifiers = modifiers,
.code_point = code_point,
.page_id = page_id,
});
}
void ConnectionFromClient::enqueue_input_event(Variant<QueuedMouseEvent, QueuedKeyboardEvent> event)
void ConnectionFromClient::enqueue_input_event(QueuedInputEvent event)
{
m_input_event_queue.enqueue(move(event));
m_input_event_queue_timer->start();

View file

@ -16,6 +16,7 @@
#include <LibWeb/CSS/PreferredColorScheme.h>
#include <LibWeb/Forward.h>
#include <LibWeb/Loader/FileRequest.h>
#include <LibWeb/Page/InputEvent.h>
#include <LibWeb/Platform/Timer.h>
#include <LibWebView/Forward.h>
#include <WebContent/Forward.h>
@ -56,13 +57,8 @@ private:
virtual void load_url(u64 page_id, URL const&) override;
virtual void load_html(u64 page_id, ByteString const&) override;
virtual void set_viewport_rect(u64 page_id, Web::DevicePixelRect const&) override;
virtual void mouse_down(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
virtual void mouse_move(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
virtual void mouse_up(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
virtual void mouse_wheel(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32, Web::DevicePixels, Web::DevicePixels) override;
virtual void doubleclick(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
virtual void key_down(u64 page_id, i32, u32, u32) override;
virtual void key_up(u64 page_id, i32, u32, u32) override;
virtual void key_event(u64 page_id, Web::KeyEvent const&) override;
virtual void mouse_event(u64 page_id, Web::MouseEvent const&) override;
virtual void add_backing_store(u64 page_id, i32 front_bitmap_id, Gfx::ShareableBitmap const& front_bitmap, i32 back_bitmap_id, Gfx::ShareableBitmap const& back_bitmap) override;
virtual void ready_to_paint(u64 page_id) override;
virtual void debug_request(u64 page_id, ByteString const&, ByteString const&) override;
@ -136,42 +132,16 @@ private:
HashMap<int, Web::FileRequest> m_requested_files {};
int last_id { 0 };
struct QueuedMouseEvent {
enum class Type {
MouseMove,
MouseDown,
MouseUp,
MouseWheel,
DoubleClick,
};
Type type {};
Web::DevicePixelPoint position {};
Web::DevicePixelPoint screen_position {};
u32 button {};
u32 buttons {};
u32 modifiers {};
Web::DevicePixels wheel_delta_x {};
Web::DevicePixels wheel_delta_y {};
struct QueuedInputEvent {
u64 page_id { 0 };
Web::InputEvent event;
size_t coalesced_event_count { 0 };
u64 page_id { 0 };
};
struct QueuedKeyboardEvent {
enum class Type {
KeyDown,
KeyUp,
};
Type type {};
i32 key {};
u32 modifiers {};
u32 code_point {};
u64 page_id { 0 };
};
void enqueue_input_event(Variant<QueuedMouseEvent, QueuedKeyboardEvent>);
void enqueue_input_event(QueuedInputEvent);
void process_next_input_event();
Queue<Variant<QueuedMouseEvent, QueuedKeyboardEvent>> m_input_event_queue;
Queue<QueuedInputEvent> m_input_event_queue;
RefPtr<Web::Platform::Timer> m_input_event_queue_timer;
};

View file

@ -7,6 +7,7 @@
#include <LibWeb/CSS/Selector.h>
#include <LibWeb/HTML/ColorPickerUpdateState.h>
#include <LibWeb/HTML/SelectedFile.h>
#include <LibWeb/Page/InputEvent.h>
#include <LibWeb/WebDriver/ExecuteScript.h>
#include <LibWebView/Attribute.h>
@ -29,14 +30,8 @@ endpoint WebContentServer
set_viewport_rect(u64 page_id, Web::DevicePixelRect rect) =|
mouse_down(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
mouse_move(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
mouse_up(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
mouse_wheel(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers, Web::DevicePixels wheel_delta_x, Web::DevicePixels wheel_delta_y) =|
doubleclick(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
key_down(u64 page_id, i32 key, u32 modifiers, u32 code_point) =|
key_up(u64 page_id, i32 key, u32 modifiers, u32 code_point) =|
key_event(u64 page_id, Web::KeyEvent event) =|
mouse_event(u64 page_id, Web::MouseEvent event) =|
debug_request(u64 page_id, ByteString request, ByteString argument) =|
get_source(u64 page_id) =|