#!/usr/bin/perl

#     Copyright (C) 2006 David M. Turner <novalis@novalis.org>
# 
#     This program is free software; you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation; either version 2 of the License, or
#     (at your option) any later version.
# 
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
# 
#     You should have received a copy of the GNU General Public License
#     along with this program; if not, write to the Free Software
#     Foundation, Inc., 51 Franklin St, Boston, MA 02110 USA


use strict;

my $start_of_post = qr{<!-- post #(\d+) -->}sm; 
my $start_of_date = qr{<img title="Old" class="inlineimg" src="search_files/post_old.gif" alt="Old" border="0"></a>}sm;
my $date = qr{(\d\d-\d\d-\d{4}, \d\d:\d\d (?:AM|PM))}sm;
my $person = qr{<a class="bigusername" href="[^"]+\?u=(\d+)">(.*?)</a>}sm;

#this one is only on the first post, looks like
my $title = qr{<!-- icon and title -->\W*<div class="smallfont">\W*<strong>(.*?)</strong>}sm;

sub degooglize {
  #removes highlights added by Google cache
  my $str = shift;
  $str =~ s{<b style="color: black; background-color: rgb\(\d+, \d+, \d+\);">([^<]+)</b>}{$1}g; #}
  return $str;
}

my $body = qr{<div id="post_message_(?:\d+)">(.*?)</div>(?:\W+)<!-- / message -->}sm;

my $page = join ("", <>);

open my $output, ">>output.txt";

my $post_title = undef;
my $post_id = -1;
my $post_user_name;
my $post_user_id;
my $post_body;
my $post_date;

my $post_count = 0;

while (1) {
  if ($page !~ /$start_of_post/g) {
    print "Recovered $post_count posts.\n";
    last;
  }
  $post_id = $1;
  $post_count += 1;

  if ($page !~ /$start_of_date/g) {
    print "PARSER ERROR: couldn't get date start in post $post_id\n";
  }
  if ($page !~ /$date/g) {
    print "PARSER ERROR: couldn't get date itself in post $post_id\n";
  }
  $post_date = $1;

  if ($page !~ /$person/g) {
    print "PARSER ERROR: couldn't get poster in post $post_id\n";
  }

  $post_user_id = $1;
  $post_user_name = $2;  

  #it's possible that the poster has an avatar.  If so, get the name from it.

  if ($post_user_name =~ /^<img/) {
    $post_user_name =~ /alt="([^"]+)'s Avatar/;
    $post_user_name = $1;
  }

  if (!$post_title) {
    if ($page !~ /$title/g) {
      print "PARSER ERROR: couldn't get title on first post $post_id\n";
      $post_title = "[TITLE LOST]";
    } else {
      $post_title = degooglize ($1);
    }
  }

  if ($page !~ /$body/gsm) {
    my $left = length ($page);
    print "PARSER ERROR: couldn't get body in post $post_id with $left left.\n"
  }

  $post_body = $1;

  $post_body =~ s/[\t\n]/ /gsm;  #we're doing a tab-delimited file, so we
                             #can't have tabs in the body.  Also,
                             #because records are delimited by
                             #newlines, we need to zap those

  $post_body = degooglize ($post_body);

  print $output "$post_id\t$post_title\t$post_date\t$post_user_id\t$post_user_name\t$post_body\n";

  # cut off page
  $page = substr ($page, pos ($page));

}