From d0d6cd4b581cb016493401e190e5b813218816e5 Mon Sep 17 00:00:00 2001 From: dakkar Date: Wed, 2 Jan 2008 21:20:15 +0000 Subject: better handling of non-existing files, and a --clean option git-svn-id: svn://luxion/repos/WebCoso/trunk@319 fcb26f47-9200-0410-b104-b98ab5b095f3 --- lib/File/Cache/Parsed.pm | 4 +- stest.pl | 292 ------------------------------------ t/fcp-01.t | 2 + t/test-site/src/bar/list.tt | 2 + webcoso.pl | 353 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 360 insertions(+), 293 deletions(-) delete mode 100644 stest.pl create mode 100644 webcoso.pl diff --git a/lib/File/Cache/Parsed.pm b/lib/File/Cache/Parsed.pm index a3843d1..bc4036d 100644 --- a/lib/File/Cache/Parsed.pm +++ b/lib/File/Cache/Parsed.pm @@ -80,7 +80,9 @@ sub get { return $self->{cache}{$filename} if exists $self->{cache}{$filename}; - my $contents=file($filename)->slurp; + my $contents=eval {file($filename)->slurp}; + + return if $@; my $pp=first {$filename =~ m{$_->[0]}} @{$self->{parsers}}; if ($pp) { diff --git a/stest.pl b/stest.pl deleted file mode 100644 index 280a0c5..0000000 --- a/stest.pl +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; -use Slay::Maker; -use File::Next; -use Path::Class; -use Template; -use File::Cache::Parsed; -use Cwd 'abs_path'; -use Text::Restructured; -use Text::Restructured::Writer::LibXML; -use XML::LibXML; -use XML::LibXSLT; -use YAML::Syck; -use Getopt::Long; - -my $SRCPATH='src'; -my $DSTPATH='dst'; -my $DSTBASEURL='/'; -my @TMPLPATH=('common/'); - -{ -my $res=GetOptions('src|s=s'=>\$SRCPATH, - 'dst|d=s'=>\$DSTPATH, - 'url|u=s'=>\$DSTBASEURL, - 'include|I=s'=>\@TMPLPATH, - ); -exit 1 unless $res; -$SRCPATH=~s{/+$}{}; -$DSTPATH=~s{/+$}{}; -$DSTBASEURL=~s{/*$}{/}; -} - -my $stash={}; -my $template_provider=Template::Provider->new({ - INCLUDE_PATH=> \@TMPLPATH, - ABSOLUTE=>1, - RELATIVE=>1, -}); -my $template=Template->new({ - LOAD_TEMPLATES=>[$template_provider], -}); -my $rest=Text::Restructured->new({},'WebCoso'); -my $xml_parser=XML::LibXML->new(); -my $xslt_proc=XML::LibXSLT->new(); -my $xpath=XML::LibXML::XPathContext->new(); -$xpath->registerNs('x', 'http://www.w3.org/1999/xhtml'); - -my $fc=File::Cache::Parsed->new(follow=>1); -$fc->add_parser(qr{\.tt2?$} => - sub { - $template->context->template($_[0]); - }); -$fc->add_parser(qr{\.rest\.txt$} => - sub { - my $dudom=$rest->Parse($_[1],$_[0]); - return Text::Restructured::Writer::LibXML - ->new->ProcessDOM($dudom); - }); -$fc->add_parser(qr{\.xml$} => - sub { $xml_parser->parse_string($_[1],$_[0]) }); -$fc->add_parser(qr{\.xslt?$} => - sub { $xslt_proc->parse_stylesheet - ($xml_parser->parse_string($_[1],$_[0])) }); -$fc->add_writer(qr{\.xml$} => - sub { $_[1]->toFile($_[0]) }); -$fc->add_parser(qr{\.ya?ml$} => - sub { Load($_[1]) }); -$fc->add_writer(qr{\.ya?ml$} => - sub { DumpFile($_[0],$_[1]) }); - - -sub dstUriFor { - my ($name)=@_; - warn "dstUriFor($name)\n"; - $name=~s{\.[^.]+\.[^.]+$}{.html}; - $name=~s{^\Q$SRCPATH\E/}{$DSTBASEURL}; - return $name; -} - -sub getTitleFor { - my ($lang,$path,$name)=@_; - - my $doc_name=$name; - $doc_name=~s{\.html$}{.du.xml}; - $doc_name=~s{/$}{/document.$lang.du.xml}; - if ($doc_name=~m{^\Q$DSTBASEURL\E}) { - $doc_name=~s{^\Q$DSTBASEURL\E}{$SRCPATH/}; - } - else { - $doc_name="$SRCPATH/$doc_name"; - } - #$doc_name=file($doc_name)->absolute(file($path)->parent); # absolutize it - warn "getTitleFor($lang,$path,$name)->$doc_name\n"; - - my $doc=$fc->get($doc_name); - unless ($doc) { - warn "No document for <$doc_name>, returning <$name>\n"; - return $name; - } - my $title=$xpath->findnodes( - q{/document/title}, - $doc); - return $title; -} - -$xslt_proc->register_function('http://webcoso.thenautilus.net/','title-for',\&getTitleFor); - -sub isLang { - my ($lang,$name)=@_; - warn "isLang($lang,$name)\n"; - return 1 if $name=~m{/$}; # assume that MultiViews on the server will work - return 1 if $name=~m{/document\.$lang(\.|$)}; - return; -} - -$stash->{dstUriFor}=\&dstUriFor; -$stash->{isLang}=\&isLang; - -sub expandTT { - my ($maker,$target,$deps,$matches)=@_; - - warn "expandTT($target,@$deps,@$matches)\n"; - - my $tmpl=$fc->get($deps->[-1]); - my $vars={ path=> $matches->[0], - language => $matches->[1], - %$stash, - }; - if (@$deps>1) { - warn "tagging as $deps->[0]\n"; - $vars->{tagged}=$fc->get($deps->[0]); - } - push @{$template_provider->include_path}, - file($deps->[-1])->parent; - $fc->put($target, - $template->context->process($tmpl, - $vars)); - pop @{$template_provider->include_path}; -} - -sub parseRST { - my ($maker,$target,$deps,$matches)=@_; - - $fc->put($target,$fc->get($deps->[-1])); -} - -sub du2html { - my ($maker,$target,$deps,$matches)=@_; - - my $du=$fc->get($deps->[-1]); - my $xslt=file($deps->[-1])->parent->file('du2html.xsl'); - $xslt=$fc->get($xslt); - my $out=$xslt->transform($du, - XML::LibXSLT::xpath_to_string( - path => $matches->[0], - language => $matches->[1], - filename => $deps->[-1], - )); - $fc->put($target,$xslt->output_string($out)); -} - -sub getTags { - my ($maker,$target,$deps,$matches)=@_; - - my %tagged; - for my $doc_name (@$deps) { - my $doc=$fc->get($doc_name); - my @tags=map {$_->textContent} - $xpath->findnodes( - q{/document/docinfo/field[field_name='tags']/field_body/*/list_item}, - $doc); - chomp for @tags; - push @{$tagged{$_}},$doc_name for @tags; - } - $fc->put($target,\%tagged); -} - -sub getChanges { - my ($maker,$target,$deps,$matches)=@_; - - print "changes: $target <- @$deps, @$matches\n"; - open my $fh,'>',$target; -} - -sub ifExists { - my ($src)=@_; - return sub { - my ($maker,$target,$matches)=@_; - my $dep=Slay::MakerRule::var_expand_dep($src,$target,$matches); - return if -e $target and ! -e $dep; - return $dep; - } -} - -sub fromTo { - my ($base,$opts)=@_; - my $iter=File::Next::files( - { - file_filter=>$opts->{files}, - descend_filter=>$opts->{dirs}, - }, - $base); - my (@ret,$file); - if (defined $opts->{transform}) { - push @ret,$opts->{transform}->($file) while $file=$iter->(); - } - else { - push @ret,$file while $file=$iter->(); - } - return @ret; -} - -{ -my %order=( - 'document.rest.tt'=>0, - 'document.rest.txt'=>1, - 'document.du.xml'=>2, -); -sub earliest { - my ($a,$b)=@_; - return $a unless $b; - return $order{$a} < $order{$b} - ? $a - : $b; -} -} - -sub keepEarliest { - my %dirs; - for my $f (@_) { - my $c=file($f); - $dirs{$c->parent}=earliest($c->basename,$dirs{$c->parent}); - } - my @ret; - while (my ($d,$f)=each %dirs) { - push @ret,file($d,$f)->stringify; - } - return @ret; -} - -my %docfiles=( - files=>sub{m{^document\.} and ($File::Next::dir !~ m{/\.svn/})}, - dirs=>sub{!m{^(tags|_webcoso)$}}); - -my $maker=Slay::Maker->new({ - rules => [ - ["$SRCPATH/tags/(**)/document.(*).rest.txt",':',"$SRCPATH/_webcoso/tags.yml",ifExists("$SRCPATH/tags/\$1/document.\$2.rest.tt"),'=',\&expandTT], - ["$SRCPATH/tags/(**)/document.(*).du.xml",':',"$SRCPATH/_webcoso/tags.yml",ifExists("$SRCPATH/tags/\$1/document.\$2.rest.txt"),'=',\&parseRST], - ["$SRCPATH/(**)/document.(*).rest.txt",':',ifExists("$SRCPATH/\$1/document.\$2.rest.tt"),'=',\&expandTT], - ["$SRCPATH/(**)/document.(*).du.xml",':',ifExists("$SRCPATH/\$1/document.\$2.rest.txt"),'=',\&parseRST], - ["$SRCPATH/()document.(*).rest.txt",':',ifExists("$SRCPATH/document.\$2.rest.tt"),'=',\&expandTT], - ["$SRCPATH/()document.(*).du.xml",':',ifExists("$SRCPATH/document.\$2.rest.txt"),'=',\&parseRST], - ["$SRCPATH/_webcoso/tags.yml",':',fromTo("$SRCPATH/",{%docfiles,transform=>sub{my $s=shift;$s=~s{\.[^.]+\.[^.]+$}{.du.xml};$s}}),'=',\&getTags], - #["$SRCPATH/_webcoso/changes.xml",':',keepEarliest(fromTo("$SRCPATH/",{%docfiles})),'=',\&getChanges], - - ["$DSTPATH/tags/(**)/document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/tags/\$1/document.\$2.du.xml",'=',\&du2html], - ["$DSTPATH/(**)/document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/\$1/document.\$2.du.xml",'=',\&du2html], - ["$DSTPATH/()document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/document.\$2.du.xml",'=',\&du2html], - ], - options => { - auto_create_dirs => 1, - #detect_no_diffs => 1, - #detect_no_size_change => 1, - #debug => 1, - }, -}); - -my @targets=(fromTo("$SRCPATH/", - { - %docfiles, - transform=>sub{ - my $src=shift; - $src=~s{\.[^.]+\.[^.]+$}{.html}; - $src=~s{^\Q$SRCPATH\E/}{$DSTPATH/}; - return $src; - }, - }), - fromTo("$SRCPATH/tags/", - { - %docfiles, - transform=>sub{ - my $src=shift; - $src=~s{\.[^.]+\.[^.]+$}{.html}; - $src=~s{^\Q$SRCPATH\E/tags/}{$DSTPATH/tags/}; - return $src; - }, - })); - -print "targets: @targets\n"; -$maker->make(@targets); - diff --git a/t/fcp-01.t b/t/fcp-01.t index 88ed9bf..7626cd4 100644 --- a/t/fcp-01.t +++ b/t/fcp-01.t @@ -112,3 +112,5 @@ is($fc->get($wr_file->stringify), 'bad', 'parsed after put'); } + +is($fc->get('nosuchfile'),undef,'return undef on non-existant files'); diff --git a/t/test-site/src/bar/list.tt b/t/test-site/src/bar/list.tt index 567b32b..fe0e939 100644 --- a/t/test-site/src/bar/list.tt +++ b/t/test-site/src/bar/list.tt @@ -3,3 +3,5 @@ [% FOR i IN [1,2,3] %] * line [% i %] [% END %] + +link: ``_ diff --git a/webcoso.pl b/webcoso.pl new file mode 100644 index 0000000..7840f53 --- /dev/null +++ b/webcoso.pl @@ -0,0 +1,353 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Slay::Maker; +use File::Next; +use Path::Class; +use Template; +use File::Cache::Parsed; +use Cwd 'abs_path'; +use Text::Restructured; +use Text::Restructured::Writer::LibXML; +use XML::LibXML; +use XML::LibXSLT; +use YAML::Syck; +use Getopt::Long; + +my $SRCPATH='src'; +my $DSTPATH='dst'; +my $DSTBASEURL='/'; +my @TMPLPATH=('common/'); +my $CLEAN=0; + +{ +my $res=GetOptions('src|s=s'=>\$SRCPATH, + 'dst|d=s'=>\$DSTPATH, + 'url|u=s'=>\$DSTBASEURL, + 'include|I=s'=>\@TMPLPATH, + 'clean'=>\$CLEAN, + ); +exit 1 unless $res; +$SRCPATH=~s{/+$}{}; +$DSTPATH=~s{/+$}{}; +$DSTBASEURL=~s{/*$}{/}; +} + +my $stash={}; +my $template_provider=Template::Provider->new({ + INCLUDE_PATH=> \@TMPLPATH, + ABSOLUTE=>1, + RELATIVE=>1, +}); +my $template=Template->new({ + LOAD_TEMPLATES=>[$template_provider], +}); + +my $rest=Text::Restructured->new( + { + D=>{ + 'file-insertion-enabled'=>0, # we use TT + generator=>0, + date=>0, + 'time'=>0, + 'source-link'=>0, + 'section-subtitles'=>1, + }, + }, + 'WebCoso'); + +my $xml_parser=XML::LibXML->new(); +my $xslt_proc=XML::LibXSLT->new(); +my $xpath=XML::LibXML::XPathContext->new(); +$xpath->registerNs('x', 'http://www.w3.org/1999/xhtml'); + +my $fc=File::Cache::Parsed->new(follow=>1); +$fc->add_parser(qr{\.tt2?$} => + sub { + $template->context->template($_[0]); + }); +$fc->add_parser(qr{\.rest\.txt$} => + sub { + my $dudom=$rest->Parse($_[1],$_[0]); + return Text::Restructured::Writer::LibXML + ->new->ProcessDOM($dudom); + }); +$fc->add_parser(qr{\.xml$} => + sub { $xml_parser->parse_string($_[1],$_[0]) }); +$fc->add_parser(qr{\.xslt?$} => + sub { $xslt_proc->parse_stylesheet + ($xml_parser->parse_string($_[1],$_[0])) }); +$fc->add_writer(qr{\.xml$} => + sub { $_[1]->toFile($_[0]) }); +$fc->add_parser(qr{\.ya?ml$} => + sub { Load($_[1]) }); +$fc->add_writer(qr{\.ya?ml$} => + sub { DumpFile($_[0],$_[1]) }); + +sub getTitleFor { + my ($lang,$path,$name)=@_; + + my $doc_name=$name; + $doc_name=~s{\.html$}{.du.xml}; + $doc_name=~s{/$}{/document.$lang.du.xml}; + if ($doc_name=~m{^\Q$DSTBASEURL\E}) { + $doc_name=~s{^\Q$DSTBASEURL\E}{$SRCPATH/}; + } + else { + $doc_name=file($doc_name)->absolute(file($path)->parent)->relative($SRCPATH); # absolutize it + $doc_name="$SRCPATH/$doc_name"; + } + warn "getTitleFor($lang,$path,$name)->$doc_name\n"; + + my $doc=$fc->get($doc_name); + unless ($doc) { + warn "No document for <$doc_name>, returning <$name>\n"; + return $name; + } + my $title=$xpath->findnodes( + q{/document/title}, + $doc); + return $title; +} + +$xslt_proc->register_function('http://webcoso.thenautilus.net/','title-for',\&getTitleFor); + +sub dstUriFor { + my ($name)=@_; + warn "dstUriFor($name)\n"; + $name=~s{\.[^.]+\.[^.]+$}{.html}; + $name=~s{^\Q$SRCPATH\E/}{$DSTBASEURL}; + return $name; +} + +sub isLang { + my ($lang,$name)=@_; + warn "isLang($lang,$name)\n"; + return 1 if $name=~m{/$}; # assume that MultiViews on the server will work + return 1 if $name=~m{/document\.$lang(\.|$)}; + return; +} + +$stash->{dstUriFor}=\&dstUriFor; +$stash->{isLang}=\&isLang; + +sub expandTT { + my ($maker,$target,$deps,$matches)=@_; + + warn "expandTT($target,@$deps,@$matches)\n"; + + my $tmpl=$fc->get($deps->[-1]); + my $vars={ path=> $matches->[0], + language => $matches->[1], + %$stash, + }; + if (@$deps>1) { + warn "tagging as $deps->[0]\n"; + $vars->{tagged}=$fc->get($deps->[0]); + } + push @{$template_provider->include_path}, + file($deps->[-1])->parent; + $fc->put($target, + $template->context->process($tmpl, + $vars)); + pop @{$template_provider->include_path}; +} + +sub parseRST { + my ($maker,$target,$deps,$matches)=@_; + + $fc->put($target,$fc->get($deps->[-1])); +} + +sub du2html { + my ($maker,$target,$deps,$matches)=@_; + + my $du=$fc->get($deps->[-1]); + my $xslt=file($deps->[-1])->parent->file('du2html.xsl'); + $xslt=$fc->get($xslt); + my $out=$xslt->transform($du, + XML::LibXSLT::xpath_to_string( + path => $matches->[0], + language => $matches->[1], + filename => $deps->[-1], + )); + $fc->put($target,$xslt->output_string($out)); +} + +sub getTags { + my ($maker,$target,$deps,$matches)=@_; + + my %tagged; + for my $doc_name (@$deps) { + my $doc=$fc->get($doc_name); + my @tags=map {$_->textContent} + $xpath->findnodes( + q{/document/docinfo/field[field_name='tags']/field_body/*/list_item}, + $doc); + chomp for @tags; + push @{$tagged{$_}},$doc_name for @tags; + } + $fc->put($target,\%tagged); +} + +sub getChanges { + my ($maker,$target,$deps,$matches)=@_; + + warn "changes: $target <- @$deps, @$matches\n"; + open my $fh,'>',$target; +} + +sub ifExists { + my ($src)=@_; + return sub { + my ($maker,$target,$matches)=@_; + my $dep=Slay::MakerRule::var_expand_dep($src,$target,$matches); + return if -e $target and ! -e $dep; + return $dep; + } +} + +sub fromTo { + my ($base,$opts)=@_; + my $iter=File::Next::files( + { + file_filter=>$opts->{files}, + descend_filter=>$opts->{dirs}, + }, + $base); + my (@ret,$file); + if (defined $opts->{transform}) { + push @ret,$opts->{transform}->($file) while $file=$iter->(); + } + else { + push @ret,$file while $file=$iter->(); + } + return @ret; +} + +{ +my %order=( + 'rest.tt'=>0, + 'rest.txt'=>1, + 'du.xml'=>2, +); +sub earliest { + my ($a,$b)=@_; + return $a unless $b; + return $order{$a} < $order{$b} + ? $a + : $b; +} + +sub keepEarliest { + my %dirs; + for my $f (@_) { + my $c=file($f); + $c->basename=~m{^document\.([^.]+)\.([^.]+\.[^.]+)$} + or die "Weird document name <$f>"; + my ($lang,$type)=($1,$2); + $dirs{$c->parent}->{$lang}=earliest($type,$dirs{$c->parent}->{$lang}); + } + my @ret; + while (my ($d,$langs)=each %dirs) { + while (my ($lang,$type)=each %$langs) { + push @ret,file($d,"document.$lang.$type")->stringify; + } + } + return @ret; +} +} + +my %docfiles=( + files=>sub{m{^document\.}}, + dirs=>sub{!m{^(tags|_webcoso|\.svn)$}}); + +if ($CLEAN) { + my %to_keep; + @to_keep{keepEarliest(fromTo($SRCPATH,{%docfiles,dirs=>sub{!m{^(_webcoso|\.svn)$}}}))}=(); + @to_keep{fromTo($SRCPATH,{files=>sub{!m{^document\.}},dirs=>sub{!m{^(_webcoso|\.svn)$}}})}=(); + + my $iter=File::Next::files({descend_filter=>sub{!m{^\.svn$}}},$SRCPATH,$DSTPATH); + while (defined (my $file=$iter->())) { + next if exists $to_keep{$file}; + unlink $file; + } + dir($DSTPATH)->rmtree; + dir($SRCPATH,'_webcoso')->rmtree; + exit 0; +} + +my $maker1st=Slay::Maker->new({ + rules => [ + ["$SRCPATH/tags/(**)/document.(*).rest.txt",':',"$SRCPATH/_webcoso/tags.yml",ifExists("$SRCPATH/tags/\$1/document.\$2.rest.tt"),'=',\&expandTT], + ["$SRCPATH/tags/(**)/document.(*).du.xml",':',"$SRCPATH/_webcoso/tags.yml",ifExists("$SRCPATH/tags/\$1/document.\$2.rest.txt"),'=',\&parseRST], + ["$SRCPATH/(**)/document.(*).rest.txt",':',ifExists("$SRCPATH/\$1/document.\$2.rest.tt"),'=',\&expandTT], + ["$SRCPATH/(**)/document.(*).du.xml",':',ifExists("$SRCPATH/\$1/document.\$2.rest.txt"),'=',\&parseRST], + ["$SRCPATH/()document.(*).rest.txt",':',ifExists("$SRCPATH/document.\$2.rest.tt"),'=',\&expandTT], + ["$SRCPATH/()document.(*).du.xml",':',ifExists("$SRCPATH/document.\$2.rest.txt"),'=',\&parseRST], + ["$SRCPATH/_webcoso/tags.yml",':',fromTo($SRCPATH,{%docfiles,transform=>sub{my $s=shift;$s=~s{\.[^.]+\.[^.]+$}{.du.xml};$s}}),'=',\&getTags], + #["$SRCPATH/_webcoso/changes.xml",':',keepEarliest(fromTo($SRCPATH,{%docfiles})),'=',\&getChanges], + ], + options => { + auto_create_dirs => 1, + #debug => 1, + }, +}); + +my $maker2nd=Slay::Maker->new({ + rules => [ + ["$DSTPATH/tags/(**)/document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/tags/\$1/document.\$2.du.xml",'=',\&du2html], + ["$DSTPATH/(**)/document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/\$1/document.\$2.du.xml",'=',\&du2html], + ["$DSTPATH/()document.(*).html",':',"$SRCPATH/_webcoso/tags.yml","$SRCPATH/document.\$2.du.xml",'=',\&du2html], + ], + options => { + auto_create_dirs => 1, + #debug => 1, + }, +}); + +my @targets1st=(fromTo("$SRCPATH/", + { + %docfiles, + transform=>sub{ + my $src=shift; + $src=~s{\.[^.]+\.[^.]+$}{.du.xml}; + return $src; + }, + }), + fromTo("$SRCPATH/tags/", + { + %docfiles, + transform=>sub{ + my $src=shift; + $src=~s{\.[^.]+\.[^.]+$}{.du.xml}; + return $src; + }, + })); + +my @targets2nd=(fromTo("$SRCPATH/", + { + %docfiles, + transform=>sub{ + my $src=shift; + $src=~s{\.[^.]+\.[^.]+$}{.html}; + $src=~s{^\Q$SRCPATH\E/}{$DSTPATH/}; + return $src; + }, + }), + fromTo("$SRCPATH/tags/", + { + %docfiles, + transform=>sub{ + my $src=shift; + $src=~s{\.[^.]+\.[^.]+$}{.html}; + $src=~s{^\Q$SRCPATH\E/tags/}{$DSTPATH/tags/}; + return $src; + }, + })); + +print "targets 1st: @targets1st\n"; +$maker1st->make(@targets1st); + +print "targets 2nd: @targets2nd\n"; +$maker2nd->make(@targets2nd); -- cgit v1.2.3