store tweets in postgres

1 files changed, 15 insertions, 39 deletions

diff --git a/tweet-archive.pl b/tweet-archive.pl
index 3c6c0b0..170c6af 100644
--- a/tweet-archive.pl
+++ b/tweet-archive.pl
@@ -1,52 +1,28 @@
 #!/usr/bin/env perl
-use 5.016;
+use 5.024;
 use strict;
 use warnings;
 use Path::Class;
-use JSON;
-use Net::Twitter;
-use ElasticSearch;
-use DateTime::Format::Strptime;
-use Data::Visitor::Callback;
-use Data::Printer;
+use JSON::MaybeXS;
+use Dakkar::TweetArchive;
+use Dakkar::TweetArchive::Store;
 
-my $conf = JSON->new->utf8->decode(
+my $json_parser = JSON::MaybeXS->new(
+    utf8 => 1,
+    relaxed => 1,
+);
+
+my $conf = $json_parser->decode(
     file(__FILE__)->parent->file('tweet-archive.conf')
         ->slurp(iomode=>'<:raw')
             // '{}'
     );
 
-my $cb = Data::Visitor::Callback->new(
-    'JSON::Boolean' => sub { say "converting a boolean"; return 0+$_ },
-);
-my $dt = DateTime::Format::Strptime->new(pattern => '%a %b %d %T %z %Y');
-
-my $nt = Net::Twitter->new(
-    traits => [qw/API::RESTv1_1 OAuth/],
-    consumer_key    => $conf->{consumer_key},
-    consumer_secret => $conf->{consumer_secret},
-);
-
-$nt->access_token($conf->{access_token});
-$nt->access_token_secret($conf->{access_token_secret});
+my $client = Dakkar::TweetArchive->new($conf);
+my $store = Dakkar::TweetArchive::Store->new($conf);
 
-my $es = ElasticSearch->new();
+my $latest_id = $store->latest_tweet_id;
 
-my $tl = $nt->home_timeline({
-    include_entities => 1,
-    trim_user => 1,
-    exclude_replies => 0,
-});
-
-for my $tweet (@$tl) {
-    $cb->visit($tweet);
-    say p $tweet;
-    my $result = $es->index(
-        index => 'tweet-archive',
-        type => 'tweet',
-        id => $tweet->{id_str},
-        timestamp => $dt->parse_datetime($tweet->{created_at})->iso8601,
-        data => $tweet,
-    );
-    say p $result;
+for my $tweet ($client->home_timeline($latest_id)->@*) {
+    $store->store_tweet($tweet);
 }