From 491dc1aeab9b445ee28f972e19a1cbb4cb9f3af7 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 20 Jan 2024 17:30:25 +0000 Subject: look at fs on demand, don't watch it ScanDir (well, fs notifications in raku) is too slow to keep up with actual fs changes (especially when e.g. a file is being downloaded) there's actually no need to watch fs changes, we can just sync the db with the file system we look at each directory --- lib/App/MediaControl.rakumod | 42 +++++---------- lib/App/MediaControl/DB.rakumod | 16 +++--- lib/App/MediaControl/FS.rakumod | 36 +++++++++++++ lib/App/MediaControl/Model.rakumod | 101 +++++++++++++++++++++++++++++++++++++ lib/App/MediaControl/Web.rakumod | 14 ++--- lib/ScanDir.rakumod | 58 --------------------- 6 files changed, 162 insertions(+), 105 deletions(-) create mode 100644 lib/App/MediaControl/FS.rakumod create mode 100644 lib/App/MediaControl/Model.rakumod delete mode 100644 lib/ScanDir.rakumod diff --git a/lib/App/MediaControl.rakumod b/lib/App/MediaControl.rakumod index d330221..f8d2360 100644 --- a/lib/App/MediaControl.rakumod +++ b/lib/App/MediaControl.rakumod @@ -1,10 +1,11 @@ use v6.d; use DB::SQLite; -use ScanDir; use Vlc::Client; use Lirc::Client; use Lirc::Commands; use App::MediaControl::DB; +use App::MediaControl::FS; +use App::MediaControl::Model; use App::MediaControl::Web; class App::MediaControl { @@ -13,6 +14,8 @@ class App::MediaControl { has Lirc::Client $!lirc-client; has Lirc::Commands $!lirc; has App::MediaControl::DB $!db; + has App::MediaControl::FS $!fs; + has App::MediaControl::Model $!model; has App::MediaControl::Web $!web; submethod TWEAK { @@ -32,43 +35,22 @@ class App::MediaControl { ), ); + $!fs .= new( + root => $!config, + extensions => $!config.Slip, + ); + + $!model .= new(:$!db,:$!fs); + $!web .= new( port => $!config, host => $!config, - :$!vlc, :$!lirc, :$!db, + :$!vlc, :$!lirc, :$!model, ); } - method !start-scan() { - my $root = $.config; - my $extensions = any($.config.Slip); - - $!db.clear-seen(); - start react { - whenever scan-dir($root) -> $item { - when $item ~~ $root {} - when $item ~~ ScanDir::End { $!db.remove-unseen(); say "scan done" } - - my $path = $item.parent.relative($root); - $path = '' if $path eq '.'; - my $name = $item.basename; - - if !$item.e { - $!db.remove-entry(:$path,:$name); - } - else { - my $is-dir = $item.d; - if $is-dir || $item.extension ~~ $extensions { - $!db.add-entry(:$path,:$name,:$is-dir); - } - } - } - } - } - method start() { $!db.ensure-schema(); - self!start-scan(); $!web.start(); } diff --git a/lib/App/MediaControl/DB.rakumod b/lib/App/MediaControl/DB.rakumod index e69efad..dd26e22 100644 --- a/lib/App/MediaControl/DB.rakumod +++ b/lib/App/MediaControl/DB.rakumod @@ -10,8 +10,8 @@ class App::MediaControl::DB { # we need an explicit LEAVE block because on 2021.10, `will # leave { .finish }` kills precomp LEAVE { .finish with $conn }; - $conn.begin; $conn.execute('PRAGMA foreign_keys=true'); + $conn.begin; KEEP { .commit with $conn }; return $code($conn) with $conn; } @@ -91,15 +91,11 @@ class App::MediaControl::DB { } } - method remove-entry(Str:D() :$path! is copy, Str:D() :$name!) { - $path ~~ s{$} = '/'; - $path ~~ s{^} = '/'; - + method remove-entry(Int:D() $id) { self!db: { - .query(q:to/END/, :$path, :$name); + .query(q:to/END/, :$id); DELETE FROM files - WHERE name=$name - AND path=$path + WHERE id=$id END } } @@ -110,7 +106,7 @@ class App::MediaControl::DB { my ($clause, @binds) = {*}; self!db: { .query(qq:to/END/,|@binds).hashes; - SELECT id, name, is_dir, watched_time + SELECT id, path, name, is_dir, watched_time FROM files WHERE parent_id $clause ORDER BY name ASC @@ -168,7 +164,7 @@ class App::MediaControl::DB { ORDER BY watched_time DESC LIMIT ? ) - SELECT files.id, files.name, files.is_dir, recent.watched_time + SELECT files.id, files.path, files.name, files.is_dir, recent.watched_time FROM files JOIN recent ON files.id=recent.id END diff --git a/lib/App/MediaControl/FS.rakumod b/lib/App/MediaControl/FS.rakumod new file mode 100644 index 0000000..b8f68d2 --- /dev/null +++ b/lib/App/MediaControl/FS.rakumod @@ -0,0 +1,36 @@ +use v6.d; + +class App::MediaControl::FS { + has IO::Path() $.root is required; + has $!extensions; + + submethod TWEAK(:$extensions) { + $!extensions = any($extensions.Slip); + } + + method get-children-of(Str $path) { + my $base = $!root.child($path); + return @() unless $base.d; + + my @children = eager $base.dir( + test => -> $f { + my $based-f = $base.child($f); + + + $based-f.d ?? $f ~~ $*SPEC.curupdir !! + ($based-f.extension ~~ $!extensions) ?? True !! + False; + }, + ); + + return @children.map( + -> $f { + %( name => $f.basename, is_dir => $f.d ); + } + ).sort({ . }); + } + + method exists(Str $path) { + return $!root.child($path).e; + } +} diff --git a/lib/App/MediaControl/Model.rakumod b/lib/App/MediaControl/Model.rakumod new file mode 100644 index 0000000..6d1c771 --- /dev/null +++ b/lib/App/MediaControl/Model.rakumod @@ -0,0 +1,101 @@ +use v6.d; +use App::MediaControl::FS; +use App::MediaControl::DB; + +class App::MediaControl::Model { + has App::MediaControl::FS $.fs is required; + has App::MediaControl::DB $.db is required; + + method get-children-of($id) { + my @db-children = self.db.get-children-of($id); + # [{id,path,name,is_dir,watched_time}] + + my $path; + if (@db-children) { + $path = @db-children[0]; # they all have the same path + } elsif ($id.defined) { + my $entry = self.db.get-entry($id); + $path = "{$entry}{$entry}"; + } else { + $path = '/'; + } + + my @fs-children = self.fs.get-children-of($path); + # [{name,is_dir}] + + my ($db-idx, $fs-idx, $changed) = 0, 0, False; + + sub add-to-db() { + self.db.add-entry( + :$path, + name => @fs-children[$fs-idx], + is-dir => @fs-children[$fs-idx]., + ); + $changed=True; + ++$fs-idx; + } + sub remove-from-db() { + self.db.remove-entry(@db-children[$db-idx]); + $changed=True; + ++$db-idx; + } + + while ($db-idx < @db-children && $fs-idx < @fs-children) { + given @db-children[$db-idx] leg @fs-children[$fs-idx] { + when Order::Same { + ++$db-idx; ++$fs-idx; + } + + when Order::Less { + remove-from-db(); + } + + when Order::More { + add-to-db(); + } + } + } + + while ($db-idx < @db-children) { + remove-from-db(); + } + + while ($fs-idx < @fs-children) { + add-to-db(); + } + + if $changed { + @db-children = self.db.get-children-of($id); + } + + return @db-children; + } + + method get-parents-of(Int:D() $id) { + return self.db.get-parents-of($id); + } + + method get-entry(Int:D() $id) { + return self.db.get-entry($id); + } + + method mark-entry-watched(Int:D() $id) { + return self.db.mark-entry-watched($id); + } + + method get-recently-watched-folders(Int:D() $limit=20) { + my @db-folders = self.db.get-recently-watched-folders($limit); + my $changed = False; + for @db-folders -> $f { + next if self.fs.exists("{$f}{$f}"); + self.db.remove-entry($f); + $changed = True; + } + + if $changed { + @db-folders = self.db.get-recently-watched-folders($limit); + } + + return @db-folders; + } +} diff --git a/lib/App/MediaControl/Web.rakumod b/lib/App/MediaControl/Web.rakumod index 77d8e05..86c748e 100644 --- a/lib/App/MediaControl/Web.rakumod +++ b/lib/App/MediaControl/Web.rakumod @@ -4,12 +4,12 @@ use Cro::HTTP::Router; use Cro::WebApp::Template; use Vlc::Client; use Lirc::Commands; -use App::MediaControl::DB; +use App::MediaControl::Model; class App::MediaControl::Web { has Vlc::Client $.vlc is required; has Lirc::Commands $.lirc is required; - has App::MediaControl::DB $.db is required; + has App::MediaControl::Model $.model is required; has Int $.port = 8080; has Str $.host = '*'; has Cro::Service $!service handles ; @@ -18,11 +18,11 @@ class App::MediaControl::Web { my $vlc = route { post -> 'play' { await self.vlc.command('pl_play') } post -> 'play', Int:D $id { - my $file = self.db.get-entry($id); + my $file = self.model.get-entry($id); await self.vlc.play-file(|%( $file:p # no comma! )); - self.db.mark-entry-watched($id); + self.model.mark-entry-watched($id); } post -> 'pause' { await self.vlc.command('pl_pause') } post -> 'stop' { await self.vlc.command('pl_stop') } @@ -46,14 +46,14 @@ class App::MediaControl::Web { my $media = route { get -> $id=Nil { - my %reply = children => @(self.db.get-children-of($id)); + my %reply = children => @(self.model.get-children-of($id)); with $id { - %reply = self.db.get-parents-of($id); + %reply = self.model.get-parents-of($id); }; content 'application/json', %reply; } get -> 'recent' { - content 'application/json', @(self.db.get-recently-watched-folders()); + content 'application/json', @(self.model.get-recently-watched-folders()); } }; diff --git a/lib/ScanDir.rakumod b/lib/ScanDir.rakumod deleted file mode 100644 index 0f33a8b..0000000 --- a/lib/ScanDir.rakumod +++ /dev/null @@ -1,58 +0,0 @@ -use v6.d; -unit module ScanDir; - -class End {}; - -sub scan-dir(*@paths --> Supply) is export { - my $s = supply { - my %watched-dirs; - - CATCH { when X::IO { }; default { warn $_ } } - - sub start-watching(IO::Path $dir) { - return unless $dir ~~ :e; - return if %watched-dirs{$dir.Str}; - %watched-dirs{$dir.Str} = True; - - whenever $dir.watch { - my $path-io = .path.IO; - emit $path-io; - when $path-io ~~ :e & :d { - add-dir($path-io) unless %watched-dirs{$path-io.Str}; - } - when $path-io ~~ :!e { - %watched-dirs{$path-io.Str}:delete - } - } - } - - sub add-dir(*@todo) { - while @todo { - my $next = @todo.shift; - - next unless $next ~~ :e & :r & :d; - start-watching($next); - - for $next.dir { - emit $_; - when .e && .d { - @todo.push($_); - start-watching($_); - } - } - } - - } - - add-dir(@paths».IO); - emit End; - }; - - # let's not return multiple events for the same path too quickly, - # otherwise the consumer will get overwhelmed when (for example) a - # large file is being written - return $s.unique( - with => sub { $^a !~~ End && $^b !~~ End && $^a eq $^b }, - expires => 0.1, - ); -} -- cgit v1.2.3