Fix pagination of Misskey responses

Misskey API, with `sinceId`, pages by `id ASC`, not descending as I
thought.

Also, allow using callbacks for paginated queries instead of
accumulating all results (much nicer for the initial db filling)

2 files changed, 18 insertions, 9 deletions

diff --git a/lib/Dakkar/Misskey.pm b/lib/Dakkar/Misskey.pm
index 327cfe8..484131f 100644
--- a/lib/Dakkar/Misskey.pm
+++ b/lib/Dakkar/Misskey.pm
@@ -5,7 +5,7 @@ use JSON::MaybeXS;
 use LWP::UserAgent;
 use Types::Standard qw(Str);
 use Types::URI qw(Uri);
-use List::Util qw(minstr);
+use List::Util qw(maxstr);
 use URI;
 use namespace::clean;
 
@@ -38,7 +38,7 @@ sub _request($self, $endpoint, $payload) {
     die $response->status_line;
 }
 
-sub _paged_request($self, $endpoint, $payload) {
+sub _paged_request($self, $endpoint, $payload, $cb) {
     my @all_results;
 
     my $page_payload = {
@@ -51,31 +51,40 @@ sub _paged_request($self, $endpoint, $payload) {
 
         last unless $result->@*;
 
-        push @all_results, $result->@*;
-        $page_payload->{untilId} = minstr(map { $_->{id} } $result->@* );
+        if ($cb) {
+            $cb->($result);
+        }
+        else {
+            push @all_results, $result->@*;
+        }
+
+        $page_payload->{sinceId} = maxstr(map { $_->{id} } $result->@* );
     }
 
     return \@all_results;
 }
 
-sub timeline($self,$options) {
+sub timeline($self,$options,$cb=undef) {
     return $self->_paged_request(
         'api/notes/timeline',
         $options,
+        $cb,
     );
 }
 
-sub followers($self,$user_id) {
+sub followers($self,$user_id,$cb=undef) {
     return $self->_paged_request(
         'api/users/followers',
         { userId => $user_id },
+        $cb,
     );
 }
 
-sub following($self,$user_id) {
+sub following($self,$user_id,$cb=undef) {
     return $self->_paged_request(
         'api/users/following',
         { userId => $user_id },
+        $cb,
     );
 }
 
diff --git a/lib/Dakkar/NotesArchive.pm b/lib/Dakkar/NotesArchive.pm
index 1fcf248..6091136 100644
--- a/lib/Dakkar/NotesArchive.pm
+++ b/lib/Dakkar/NotesArchive.pm
@@ -26,13 +26,13 @@ sub _build_client($self) {
     });
 }
 
-sub timeline($self, $since_id) {
+sub timeline($self, $since_id, $cb=undef) {
     return $self->client->timeline({
         maybe sinceId => $since_id,
         includeMyRenotes => \1,
         includeLocalRenotes => \1,
         includeRenotedMyNotes => \0,
-    });
+    }, $cb);
 }
 
 sub following($self) {