From 9dbd8be35891bb83abad647f9f488aa9a64722da Mon Sep 17 00:00:00 2001
From: dakkar <dakkar@thenautilus.net>
Date: Fri, 24 Feb 2023 12:37:42 +0000
Subject: Fix pagination of Misskey responses

Misskey API, with `sinceId`, pages by `id ASC`, not descending as I
thought.

Also, allow using callbacks for paginated queries instead of
accumulating all results (much nicer for the initial db filling)
---
 lib/Dakkar/Misskey.pm      | 23 ++++++++++++++++-------
 lib/Dakkar/NotesArchive.pm |  4 ++--
 tweet-archive.pl           | 11 +++++++----
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/lib/Dakkar/Misskey.pm b/lib/Dakkar/Misskey.pm
index 327cfe8..484131f 100644
--- a/lib/Dakkar/Misskey.pm
+++ b/lib/Dakkar/Misskey.pm
@@ -5,7 +5,7 @@ use JSON::MaybeXS;
 use LWP::UserAgent;
 use Types::Standard qw(Str);
 use Types::URI qw(Uri);
-use List::Util qw(minstr);
+use List::Util qw(maxstr);
 use URI;
 use namespace::clean;
 
@@ -38,7 +38,7 @@ sub _request($self, $endpoint, $payload) {
     die $response->status_line;
 }
 
-sub _paged_request($self, $endpoint, $payload) {
+sub _paged_request($self, $endpoint, $payload, $cb) {
     my @all_results;
 
     my $page_payload = {
@@ -51,31 +51,40 @@ sub _paged_request($self, $endpoint, $payload) {
 
         last unless $result->@*;
 
-        push @all_results, $result->@*;
-        $page_payload->{untilId} = minstr(map { $_->{id} } $result->@* );
+        if ($cb) {
+            $cb->($result);
+        }
+        else {
+            push @all_results, $result->@*;
+        }
+
+        $page_payload->{sinceId} = maxstr(map { $_->{id} } $result->@* );
     }
 
     return \@all_results;
 }
 
-sub timeline($self,$options) {
+sub timeline($self,$options,$cb=undef) {
     return $self->_paged_request(
         'api/notes/timeline',
         $options,
+        $cb,
     );
 }
 
-sub followers($self,$user_id) {
+sub followers($self,$user_id,$cb=undef) {
     return $self->_paged_request(
         'api/users/followers',
         { userId => $user_id },
+        $cb,
     );
 }
 
-sub following($self,$user_id) {
+sub following($self,$user_id,$cb=undef) {
     return $self->_paged_request(
         'api/users/following',
         { userId => $user_id },
+        $cb,
     );
 }
 
diff --git a/lib/Dakkar/NotesArchive.pm b/lib/Dakkar/NotesArchive.pm
index 1fcf248..6091136 100644
--- a/lib/Dakkar/NotesArchive.pm
+++ b/lib/Dakkar/NotesArchive.pm
@@ -26,13 +26,13 @@ sub _build_client($self) {
     });
 }
 
-sub timeline($self, $since_id) {
+sub timeline($self, $since_id, $cb=undef) {
     return $self->client->timeline({
         maybe sinceId => $since_id,
         includeMyRenotes => \1,
         includeLocalRenotes => \1,
         includeRenotedMyNotes => \0,
-    });
+    }, $cb);
 }
 
 sub following($self) {
diff --git a/tweet-archive.pl b/tweet-archive.pl
index 1a7232b..38bb184 100644
--- a/tweet-archive.pl
+++ b/tweet-archive.pl
@@ -63,11 +63,14 @@ if ($conf->{misskey}) {
     $client->client->ua->add_handler( response_done => sub { push @responses, $_[0]; return } );
 
     try {
-        my $latest_id = $store->latest_note_id;
+        my $latest_id = $store->latest_note_id // '0';
 
-        for my $note ($client->timeline($latest_id)->@*) {
-            $store->store_note($note);
-        }
+        $client->timeline(
+            $latest_id,
+            sub($notes) {
+                $store->store_note($_) for $notes->@*;
+            },
+        );
 
         $store->store_misskey_following($client->following);
         $store->store_misskey_followers($client->followers);
-- 
cgit v1.2.3