Skip to content

R
rustdesk
Commit 7c5a136b authored by csf's avatar csf
Browse files
Options

mobile & web rgba stream

parent 9ecacadd
Hide whitespace changes
Inline Side-by-side
Showing with 76 additions and 130 deletions
flutter/lib/models/model.dart
View file @ 7c5a136b
......@@ -17,12 +17,11 @@ import '../widgets/overlay.dart';
import 'native_model.dart' if (dart.library.html) 'web_model.dart';
typedef HandleMsgBox = void Function(Map<String, dynamic> evt, String id);
bool _waitForImage = false;
class FfiModel with ChangeNotifier {
PeerInfo _pi = PeerInfo();
Display _display = Display();
var _decoding = false;
bool _waitForImage = false;
var _inputBlocked = false;
final _permissions = Map<String, bool>();
bool? _secure;
......@@ -122,7 +121,6 @@ class FfiModel with ChangeNotifier {
void updateEventListener(String peerId) {
final void Function(Map<String, dynamic>) cb = (evt) {
var pos;
var name = evt['name'];
if (name == 'msgbox') {
handleMsgBox(evt, peerId);
......@@ -138,7 +136,7 @@ class FfiModel with ChangeNotifier {
} else if (name == 'cursor_id') {
FFI.cursorModel.updateCursorId(evt);
} else if (name == 'cursor_position') {
pos = evt;
FFI.cursorModel.updateCursorPosition(evt);
} else if (name == 'clipboard') {
Clipboard.setData(ClipboardData(text: evt['content']));
} else if (name == 'permission') {
......@@ -165,31 +163,6 @@ class FfiModel with ChangeNotifier {
} else if (name == 'on_client_remove') {
FFI.serverModel.onClientRemove(evt);
}
if (pos != null) FFI.cursorModel.updateCursorPosition(pos);
if (!_decoding) {
var rgba = PlatformFFI.getRgba();
if (rgba != null) {
if (_waitForImage) {
_waitForImage = false;
SmartDialog.dismiss();
}
_decoding = true;
final pid = FFI.id;
ui.decodeImageFromPixels(rgba, _display.width, _display.height,
isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888,
(image) {
PlatformFFI.clearRgbaFrame();
_decoding = false;
if (FFI.id != pid) return;
try {
// my throw exception, because the listener maybe already dispose
FFI.imageModel.update(image);
} catch (e) {
print('update image: $e');
}
});
}
}
};
PlatformFFI.setEventCallback(cb);
}
......@@ -284,6 +257,29 @@ class ImageModel with ChangeNotifier {
ui.Image? get image => _image;
ImageModel() {
PlatformFFI.setRgbaCallback((rgba) {
if (_waitForImage) {
_waitForImage = false;
SmartDialog.dismiss();
}
final pid = FFI.id;
ui.decodeImageFromPixels(
rgba,
FFI.ffiModel.display.width,
FFI.ffiModel.display.height,
isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, (image) {
if (FFI.id != pid) return;
try {
// my throw exception, because the listener maybe already dispose
FFI.imageModel.update(image);
} catch (e) {
print('update image: $e');
}
});
});
}
void update(ui.Image? image) {
if (_image == null && image != null) {
if (isDesktop) {
......
flutter/lib/models/native_model.dart
View file @ 7c5a136b
......@@ -19,8 +19,6 @@ class RgbaFrame extends Struct {
typedef F2 = Pointer<Utf8> Function(Pointer<Utf8>, Pointer<Utf8>);
typedef F3 = void Function(Pointer<Utf8>, Pointer<Utf8>);
typedef F4 = void Function(Pointer<RgbaFrame>);
typedef F5 = Pointer<RgbaFrame> Function();
class PlatformFFI {
static Pointer<RgbaFrame>? _lastRgbaFrame;
......@@ -28,23 +26,8 @@ class PlatformFFI {
static String _homeDir = '';
static F2? _getByName;
static F3? _setByName;
static F4? _freeRgba;
static F5? _getRgba;
static void Function(Map<String, dynamic>)? _eventCallback;
static void clearRgbaFrame() {
if (_lastRgbaFrame != null &&
_lastRgbaFrame != nullptr &&
_freeRgba != null) _freeRgba!(_lastRgbaFrame!);
}
static Uint8List? getRgba() {
if (_getRgba == null) return null;
_lastRgbaFrame = _getRgba!();
if (_lastRgbaFrame == null || _lastRgbaFrame == nullptr) return null;
final ref = _lastRgbaFrame!.ref;
return Uint8List.sublistView(ref.data.asTypedList(ref.len));
}
static void Function(Uint8List)? _rgbaCallback;
static Future<String> getVersion() async {
PackageInfo packageInfo = await PackageInfo.fromPlatform();
......@@ -85,9 +68,6 @@ class PlatformFFI {
_setByName =
dylib.lookupFunction<Void Function(Pointer<Utf8>, Pointer<Utf8>), F3>(
'set_by_name');
_freeRgba = dylib
.lookupFunction<Void Function(Pointer<RgbaFrame>), F4>('free_rgba');
_getRgba = dylib.lookupFunction<F5, F5>('get_rgba');
_dir = (await getApplicationDocumentsDirectory()).path;
_startListenEvent(RustdeskImpl(dylib));
try {
......@@ -119,23 +99,38 @@ class PlatformFFI {
version = await getVersion();
}
static void _startListenEvent(RustdeskImpl rustdeskImpl) async {
await for (final message in rustdeskImpl.startEventStream()) {
if (_eventCallback != null) {
try {
Map<String, dynamic> event = json.decode(message);
_eventCallback!(event);
} catch (e) {
print('json.decode fail(): $e');
static void _startListenEvent(RustdeskImpl rustdeskImpl) {
() async {
await for (final message in rustdeskImpl.startEventStream()) {
if (_eventCallback != null) {
try {
Map<String, dynamic> event = json.decode(message);
_eventCallback!(event);
} catch (e) {
print('json.decode fail(): $e');
}
}
}
}
}();
() async {
await for (final rgba in rustdeskImpl.startRgbaStream()) {
if (_rgbaCallback != null) {
_rgbaCallback!(rgba);
} else {
rgba.clear();
}
}
}();
}
static void setEventCallback(void Function(Map<String, dynamic>) fun) async {
_eventCallback = fun;
}
static void setRgbaCallback(void Function(Uint8List) fun) async {
_rgbaCallback = fun;
}
static void startDesktopWebListener() {}
static void stopDesktopWebListener() {}
......
flutter/lib/models/web_model.dart
View file @ 7c5a136b
......@@ -10,12 +10,6 @@ final List<StreamSubscription<MouseEvent>> mouseListeners = [];
final List<StreamSubscription<KeyboardEvent>> keyListeners = [];
class PlatformFFI {
static void clearRgbaFrame() {}
static Uint8List? getRgba() {
return context.callMethod('getRgba');
}
static String getByName(String name, [String arg = '']) {
return context.callMethod('getByName', [name, arg]);
}
......@@ -31,7 +25,7 @@ class PlatformFFI {
version = getByName('version');
}
static void setEventCallback(void Function(Map<String, dynamic>) fun) async {
static void setEventCallback(void Function(Map<String, dynamic>) fun) {
context["onGlobalEvent"] = (String message) {
try {
Map<String, dynamic> event = json.decode(message);
......@@ -42,6 +36,14 @@ class PlatformFFI {
};
}
static void setRgbaCallback(void Function(Uint8List) fun) {
context["onRgba"] = (Uint8List? rgba) {
if (rgba != null) {
fun(rgba);
}
};
}
static void startDesktopWebListener() {
mouseListeners.add(
window.document.onContextMenu.listen((evt) => evt.preventDefault()));
......
flutter/web/js/src/globals.js
View file @ 7c5a136b
......@@ -6,14 +6,7 @@ import { checkIfRetry, version } from "./gen_js_from_hbb";
import { initZstd, translate } from "./common";
import PCMPlayer from "pcm-player";
var currentFrame = undefined;
window.curConn = undefined;
window.getRgba = () => {
const tmp = currentFrame;
currentFrame = undefined;
return tmp || null;
}
window.isMobile = () => {
return /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|ipad|iris|kindle|Android|Silk|lge |maemo|midp|mmp|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino/i.test(navigator.userAgent)
|| /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(navigator.userAgent.substr(0, 4));
......@@ -81,7 +74,7 @@ export function draw(frame) {
for (let i = 0; i < size; i += row) {
flipPixels.set(pixels.subarray(i, i + row), end - i);
}
currentFrame = flipPixels;
onRgba(flipPixels);
testSpeed[1] += new Date().getTime() - tm0;
testSpeed[0] += 1;
if (testSpeed[0] > 30) {
......@@ -116,7 +109,6 @@ export function getConn() {
}
export async function startConn(id) {
currentFrame = undefined;
setByName('remote_id', id);
await curConn.start(id);
}
......@@ -124,7 +116,6 @@ export async function startConn(id) {
export function close() {
getConn()?.close();
setConn(undefined);
currentFrame = undefined;
}
export function newConn() {
......@@ -339,7 +330,7 @@ export function playAudio(packet) {
window.init = async () => {
if (yuvWorker) {
yuvWorker.onmessage = (e) => {
currentFrame = e.data;
onRgba(e.data);
}
}
opusWorker.onmessage = (e) => {
......
src/mobile.rs
View file @ 7c5a136b
use crate::client::*;
use flutter_rust_bridge::StreamSink;
use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer};
use hbb_common::{
allow_err,
compress::decompress,
......@@ -24,7 +24,8 @@ use std::{
lazy_static::lazy_static! {
static ref SESSION: Arc<RwLock<Option<Session>>> = Default::default();
pub static ref EVENT_STREAM: RwLock<Option<StreamSink<String>>> = Default::default(); // rust to dart channel
pub static ref EVENT_STREAM: RwLock<Option<StreamSink<String>>> = Default::default(); // rust to dart event channel
pub static ref RGBA_STREAM: RwLock<Option<StreamSink<ZeroCopyBuffer<Vec<u8>>>>> = Default::default(); // rust to dart rgba (big u8 list) channel
}
#[derive(Clone, Default)]
......@@ -33,7 +34,6 @@ pub struct Session {
sender: Arc<RwLock<Option<mpsc::UnboundedSender<Data>>>>,
lc: Arc<RwLock<LoginConfigHandler>>,
events2ui: Arc<RwLock<VecDeque<String>>>,
rgba: Arc<RwLock<Option<Vec<u8>>>>,
}
impl Session {
......@@ -89,14 +89,6 @@ impl Session {
}
}
pub fn rgba() -> Option<Vec<u8>> {
if let Some(session) = SESSION.read().unwrap().as_ref() {
session.rgba.write().unwrap().take()
} else {
None
}
}
pub fn pop_event() -> Option<String> {
if let Some(session) = SESSION.read().unwrap().as_ref() {
session.events2ui.write().unwrap().pop_front()
......@@ -607,8 +599,11 @@ impl Connection {
if !self.first_frame {
self.first_frame = true;
}
if let Ok(true) = self.video_handler.handle_frame(vf) {
*self.session.rgba.write().unwrap() = Some(self.video_handler.rgb.clone());
if let (Ok(true), Some(s)) = (
self.video_handler.handle_frame(vf),
RGBA_STREAM.read().unwrap().as_ref(),
) {
s.add(ZeroCopyBuffer(self.video_handler.rgb.clone()));
}
}
Some(message::Union::hash(hash)) => {
......
src/mobile_ffi.rs
View file @ 7c5a136b
use crate::client::file_trait::FileManager;
use crate::mobile::connection_manager::{self, get_clients_length, get_clients_state};
use crate::mobile::{self, make_fd_to_json, Session};
use flutter_rust_bridge::StreamSink;
use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer};
use hbb_common::ResultType;
use hbb_common::{
config::{self, Config, LocalConfig, PeerConfig, ONLINE},
......@@ -40,6 +40,11 @@ pub fn start_event_stream(s: StreamSink<String>) -> ResultType<()> {
Ok(())
}
pub fn start_rgba_stream(s: StreamSink<ZeroCopyBuffer<Vec<u8>>>) -> ResultType<()> {
let _ = mobile::RGBA_STREAM.write().unwrap().insert(s);
Ok(())
}
#[no_mangle]
unsafe extern "C" fn get_by_name(name: *const c_char, arg: *const c_char) -> *const c_char {
let mut res = "".to_owned();
......@@ -359,7 +364,7 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) {
m.get("file_num"),
m.get("need_override"),
m.get("remember"),
m.get("is_upload")
m.get("is_upload"),
) {
Session::set_confirm_override_file(
id.parse().unwrap_or(0),
......@@ -507,44 +512,6 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) {
}
}
#[repr(C)]
struct RgbaFrame {
len: u32,
data: *mut u8,
}
#[no_mangle]
unsafe extern "C" fn get_rgba() -> *mut RgbaFrame {
if let Some(mut vec) = Session::rgba() {
if vec.is_empty() {
return std::ptr::null_mut();
}
assert!(vec.len() == vec.capacity());
vec.shrink_to_fit();
let data = vec.as_mut_ptr();
let len = vec.len();
std::mem::forget(vec);
Box::into_raw(Box::new(RgbaFrame {
len: len as _,
data,
}))
} else {
std::ptr::null_mut()
}
}
#[no_mangle]
extern "C" fn free_rgba(f: *mut RgbaFrame) {
if f.is_null() {
return;
}
unsafe {
let len = (*f).len as usize;
drop(Vec::from_raw_parts((*f).data, len, len));
Box::from_raw(f);
}
}
#[cfg(target_os = "android")]
pub mod server_side {
use hbb_common::{config::Config, log};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment