look at fs on demand, don't watch it

ScanDir (well, fs notifications in raku) is too slow to keep up with
actual fs changes (especially when e.g. a file is being downloaded)

there's actually no need to watch fs changes, we can just sync the db
with the file system we look at each directory

6 files changed, 162 insertions, 105 deletions

diff --git a/lib/App/MediaControl.rakumod b/lib/App/MediaControl.rakumod
index d330221..f8d2360 100644
--- a/lib/App/MediaControl.rakumod
+++ b/lib/App/MediaControl.rakumod
@@ -1,10 +1,11 @@
 use v6.d;
 use DB::SQLite;
-use ScanDir;
 use Vlc::Client;
 use Lirc::Client;
 use Lirc::Commands;
 use App::MediaControl::DB;
+use App::MediaControl::FS;
+use App::MediaControl::Model;
 use App::MediaControl::Web;
 
 class App::MediaControl {
@@ -13,6 +14,8 @@ class App::MediaControl {
     has Lirc::Client $!lirc-client;
     has Lirc::Commands $!lirc;
     has App::MediaControl::DB $!db;
+    has App::MediaControl::FS $!fs;
+    has App::MediaControl::Model $!model;
     has App::MediaControl::Web $!web;
 
     submethod TWEAK {
@@ -32,43 +35,22 @@ class App::MediaControl {
             ),
         );
 
+        $!fs .= new(
+            root => $!config<media><root>,
+            extensions => $!config<media><extensions>.Slip,
+        );
+
+        $!model .= new(:$!db,:$!fs);
+
         $!web .= new(
             port => $!config<server><port>,
 	    host => $!config<server><host>,
-            :$!vlc, :$!lirc, :$!db,
+            :$!vlc, :$!lirc, :$!model,
         );
     }
 
-    method !start-scan() {
-        my $root = $.config<media><root>;
-        my $extensions = any($.config<media><extensions>.Slip);
-
-        $!db.clear-seen();
-        start react {
-            whenever scan-dir($root) -> $item {
-                when $item ~~ $root {}
-                when $item ~~ ScanDir::End { $!db.remove-unseen(); say "scan done" }
-
-                my $path = $item.parent.relative($root);
-                $path = '' if $path eq '.';
-                my $name = $item.basename;
-
-                if !$item.e {
-                    $!db.remove-entry(:$path,:$name);
-                }
-                else {
-                    my $is-dir = $item.d;
-                    if $is-dir || $item.extension ~~ $extensions {
-                        $!db.add-entry(:$path,:$name,:$is-dir);
-                    }
-                }
-            }
-        }
-    }
-
     method start() {
         $!db.ensure-schema();
-        self!start-scan();
         $!web.start();
     }
 
diff --git a/lib/App/MediaControl/DB.rakumod b/lib/App/MediaControl/DB.rakumod
index e69efad..dd26e22 100644
--- a/lib/App/MediaControl/DB.rakumod
+++ b/lib/App/MediaControl/DB.rakumod
@@ -10,8 +10,8 @@ class App::MediaControl::DB {
             # we need an explicit LEAVE block because on 2021.10, `will
             # leave { .finish }` kills precomp
             LEAVE { .finish with $conn };
-            $conn.begin;
             $conn.execute('PRAGMA foreign_keys=true');
+            $conn.begin;
             KEEP { .commit with $conn };
             return $code($conn) with $conn;
         }
@@ -91,15 +91,11 @@ class App::MediaControl::DB {
         }
     }
 
-    method remove-entry(Str:D() :$path! is copy, Str:D() :$name!) {
-        $path ~~ s{<!after '/'>$} = '/';
-        $path ~~ s{<!before '/'>^} = '/';
-
+    method remove-entry(Int:D() $id) {
         self!db: {
-            .query(q:to/END/, :$path, :$name);
+            .query(q:to/END/, :$id);
             DELETE FROM files
-            WHERE name=$name
-              AND path=$path
+            WHERE id=$id
             END
         }
     }
@@ -110,7 +106,7 @@ class App::MediaControl::DB {
         my ($clause, @binds) = {*};
         self!db: {
             .query(qq:to/END/,|@binds).hashes;
-            SELECT id, name, is_dir, watched_time
+            SELECT id, path, name, is_dir, watched_time
             FROM files
             WHERE parent_id $clause
             ORDER BY name ASC
@@ -168,7 +164,7 @@ class App::MediaControl::DB {
               ORDER BY watched_time DESC
               LIMIT ?
             )
-            SELECT files.id, files.name, files.is_dir, recent.watched_time
+            SELECT files.id, files.path, files.name, files.is_dir, recent.watched_time
             FROM files
             JOIN recent ON files.id=recent.id
             END
diff --git a/lib/App/MediaControl/FS.rakumod b/lib/App/MediaControl/FS.rakumod
new file mode 100644
index 0000000..b8f68d2
--- /dev/null
+++ b/lib/App/MediaControl/FS.rakumod
@@ -0,0 +1,36 @@
+use v6.d;
+
+class App::MediaControl::FS {
+    has IO::Path() $.root is required;
+    has $!extensions;
+
+    submethod TWEAK(:$extensions) {
+        $!extensions = any($extensions.Slip);
+    }
+
+    method get-children-of(Str $path) {
+        my $base = $!root.child($path);
+        return @() unless $base.d;
+        
+        my @children = eager $base.dir(
+            test => -> $f {
+                my $based-f = $base.child($f);
+
+
+                $based-f.d ?? $f ~~ $*SPEC.curupdir !!
+                ($based-f.extension ~~ $!extensions) ?? True !!
+                False;
+            },
+        );
+
+        return @children.map(
+            -> $f {
+                %( name => $f.basename, is_dir => $f.d );
+            }
+        ).sort({ .<name> });
+    }
+
+    method exists(Str $path) {
+        return $!root.child($path).e;
+    }
+}
diff --git a/lib/App/MediaControl/Model.rakumod b/lib/App/MediaControl/Model.rakumod
new file mode 100644
index 0000000..6d1c771
--- /dev/null
+++ b/lib/App/MediaControl/Model.rakumod
@@ -0,0 +1,101 @@
+use v6.d;
+use App::MediaControl::FS;
+use App::MediaControl::DB;
+
+class App::MediaControl::Model {
+    has App::MediaControl::FS $.fs is required;
+    has App::MediaControl::DB $.db is required;
+
+    method get-children-of($id) {
+        my @db-children = self.db.get-children-of($id);
+        # [{id,path,name,is_dir,watched_time}]
+
+        my $path;
+        if (@db-children) {
+            $path = @db-children[0]<path>; # they all have the same path
+        } elsif ($id.defined) {
+            my $entry = self.db.get-entry($id);
+            $path = "{$entry<path>}{$entry<name>}";
+        } else {
+            $path = '/';
+        }
+
+        my @fs-children = self.fs.get-children-of($path);
+        # [{name,is_dir}]
+
+        my ($db-idx, $fs-idx, $changed) = 0, 0, False;
+
+        sub add-to-db() {
+            self.db.add-entry(
+                :$path,
+                name => @fs-children[$fs-idx]<name>,
+                is-dir => @fs-children[$fs-idx].<is_dir>,
+            );
+            $changed=True;
+            ++$fs-idx;
+        }
+        sub remove-from-db() {
+            self.db.remove-entry(@db-children[$db-idx]<id>);
+            $changed=True;
+            ++$db-idx;
+        }
+        
+        while ($db-idx < @db-children && $fs-idx < @fs-children) {
+            given @db-children[$db-idx]<name> leg @fs-children[$fs-idx]<name> {
+                when Order::Same {
+                    ++$db-idx; ++$fs-idx;
+                }
+
+                when Order::Less {
+                    remove-from-db();
+                }
+
+                when Order::More {
+                    add-to-db();
+                }
+            }
+        }
+
+        while ($db-idx < @db-children) {
+            remove-from-db();
+        }
+
+        while ($fs-idx < @fs-children) {
+            add-to-db();
+        }
+
+        if $changed {
+            @db-children = self.db.get-children-of($id);
+        }
+
+        return @db-children;
+    }
+
+    method get-parents-of(Int:D() $id) {
+        return self.db.get-parents-of($id);
+    }
+
+    method get-entry(Int:D() $id) {
+        return self.db.get-entry($id);
+    }
+
+    method mark-entry-watched(Int:D() $id) {
+        return self.db.mark-entry-watched($id);
+    }
+
+    method get-recently-watched-folders(Int:D() $limit=20) {
+        my @db-folders = self.db.get-recently-watched-folders($limit);
+        my $changed = False;
+        for @db-folders -> $f {
+            next if self.fs.exists("{$f<path>}{$f<name>}");
+            self.db.remove-entry($f<id>);
+            $changed = True;
+        }
+
+        if $changed {
+            @db-folders = self.db.get-recently-watched-folders($limit);
+        }
+
+        return @db-folders;
+    }
+}
diff --git a/lib/App/MediaControl/Web.rakumod b/lib/App/MediaControl/Web.rakumod
index 77d8e05..86c748e 100644
--- a/lib/App/MediaControl/Web.rakumod
+++ b/lib/App/MediaControl/Web.rakumod
@@ -4,12 +4,12 @@ use Cro::HTTP::Router;
 use Cro::WebApp::Template;
 use Vlc::Client;
 use Lirc::Commands;
-use App::MediaControl::DB;
+use App::MediaControl::Model;
 
 class App::MediaControl::Web {
     has Vlc::Client $.vlc is required;
     has Lirc::Commands $.lirc is required;
-    has App::MediaControl::DB $.db is required;
+    has App::MediaControl::Model $.model is required;
     has Int $.port = 8080;
     has Str $.host = '*';
     has Cro::Service $!service handles <stop>;
@@ -18,11 +18,11 @@ class App::MediaControl::Web {
         my $vlc = route {
             post -> 'play' { await self.vlc.command('pl_play') }
             post -> 'play', Int:D $id {
-                my $file = self.db.get-entry($id);
+                my $file = self.model.get-entry($id);
                 await self.vlc.play-file(|%(
                     $file<path name>:p # no comma!
                 ));
-                self.db.mark-entry-watched($id);
+                self.model.mark-entry-watched($id);
             }
             post -> 'pause' { await self.vlc.command('pl_pause') }
             post -> 'stop' { await self.vlc.command('pl_stop') }
@@ -46,14 +46,14 @@ class App::MediaControl::Web {
 
         my $media = route {
             get -> $id=Nil {
-                my %reply = children => @(self.db.get-children-of($id));
+                my %reply = children => @(self.model.get-children-of($id));
                 with $id {
-                    %reply<parents> = self.db.get-parents-of($id);
+                    %reply<parents> = self.model.get-parents-of($id);
                 };
                 content 'application/json', %reply;
             }
             get -> 'recent' {
-                content 'application/json', @(self.db.get-recently-watched-folders());
+                content 'application/json', @(self.model.get-recently-watched-folders());
             }
         };
 
diff --git a/lib/ScanDir.rakumod b/lib/ScanDir.rakumod
deleted file mode 100644
index 0f33a8b..0000000
--- a/lib/ScanDir.rakumod
+++ /dev/null
@@ -1,58 +0,0 @@
-use v6.d;
-unit module ScanDir;
- 
-class End {};
- 
-sub scan-dir(*@paths --> Supply) is export {
-    my $s = supply {
-        my %watched-dirs;
- 
-        CATCH { when X::IO { }; default { warn $_ } }
- 
-        sub start-watching(IO::Path $dir) {
-            return unless $dir ~~ :e;
-            return if %watched-dirs{$dir.Str};
-            %watched-dirs{$dir.Str} = True;
-            
-            whenever $dir.watch {
-                my $path-io = .path.IO;
-                emit $path-io;
-                when $path-io ~~ :e & :d {
-                    add-dir($path-io) unless %watched-dirs{$path-io.Str};
-                }
-                when $path-io ~~ :!e {
-                    %watched-dirs{$path-io.Str}:delete
-                }
-            }
-        }
- 
-        sub add-dir(*@todo) {
-            while @todo {
-                my $next = @todo.shift;
-
-                next unless $next ~~ :e & :r & :d;
-                start-watching($next);
- 
-                for $next.dir {
-                    emit $_;
-                    when .e && .d {
-                        @todo.push($_);
-                        start-watching($_);
-                    }
-                }
-            }
- 
-        }
- 
-        add-dir(@paths».IO);
-        emit End;
-    };
-
-    # let's not return multiple events for the same path too quickly,
-    # otherwise the consumer will get overwhelmed when (for example) a
-    # large file is being written
-    return $s.unique(
-        with => sub { $^a !~~ End && $^b !~~ End && $^a eq $^b },
-        expires => 0.1,
-    );
-}