###################################################################
# Function: Prepare data for annation
# Author: Jing Zhang
# Date: 2008.5.16
#####################################################################
package data_prepare;

require Exporter;
@ISA = qw(Exporter);
@EXPORT = qw(
  process_raw_data
);
use strict;

######################################
#functions
#input: the original files
#output: the data file for annotation

sub process_raw_data
{
  my ($dir, $outputfile) = @_;
  my $backup = $/;
  $/ = undef;
  # read all the files under the dir
  opendir dirstream, $dir;
  my @files = readdir(dirstream);
  closedir dirstream;
  shift(@files);
  shift(@files);

  my %paper_index;
  my @paper_array;

  # store the downloaded titles of files
  my %download_papers;
  for( my $i = 0 ; $i < scalar(@files) ; $i ++ )
  {
    my $title = $files[$i];
    $title=~s/\.(txt|TXT)//;
    $title=trim_file_name($title);
    $download_papers{$title} = 1;
  }

  print "Document Number:", scalar(keys(%download_papers)),"\n";
  my $pair_number;
  my $feature_number;
  # parse cited information
  for( my $i = 0 ; $i < scalar(@files) ; $i ++ )
  {
    my $file = $files[$i];


    # read the content of the paper end with ".txt"
    next if($file !~/\.(txt|TXT)$/);
    my $filepath = $dir."/".$file;
    open instream, $filepath;
    my $content = <instream>;
    close instream;

    # replace disordered characters
    $content =trim_content($content);
############################## extract features for type #######################
#    while($content=~/.*\[(\d{1,3}|(?:\d{1,3},\s*)+\d{1,3})\]<(\d)>.*?\.\s*however\s*,/gi)
#    {
#       $feature_number++;
#    }
############################## extract features for type #######################


##############################extract abstract#######################
    my $abstract;
    if($content=~/<abstract>(.*?)<\/abstract>/s)
    {
      $abstract = $1;
    }else{
      $abstract = $content;
    }
    $abstract =~s/abstract//;
    $abstract =~s/fififi//;

    $abstract = trim_annotation($abstract);
##############################extract abstract#######################

    # split sentences
    my @line_array = split_sentence($content);

    # parse paper title from file name
    my $title = $file;
    $title=~s/\.(txt|TXT)//;
    $title=trim_file_name($title);

    # put paper information into a hashtable and an array
    my $index = $paper_index{$title};
    if($index eq undef)
    {
      $index = scalar(@paper_array);

      $paper_index{$title} = $index;
      my %paper_hash;
      $paper_hash{"index"} = $index;
      $paper_hash{"title"} = $title;
      $paper_hash{"abstract"} = $abstract;
      push @paper_array, \%paper_hash;
    }else{
      my $paper_hash = $paper_array[$index];
      $$paper_hash{"abstract"} = $abstract;
    }


    # parse cited paper number, cited paper type, and the corresponding words in the current paper
    my %cited_hash;
    for( my $i = 0 ; $i < scalar(@line_array);$i++)
    {
      my $line = $line_array[$i];
      #while($line=~/\[(\d{1,3}|(?:\d{1,3},\s*)+\d{1,3})\]<(\d)>/g)
      while($line=~/\[(\d{1,3}|(?:\d{1,3},\s*)+\d{1,3})\]/g)
      {
        my $cited_num_str = $1;
        my $cited_type = $2;
        my @cited_nums = split(/,/,$cited_num_str);
        my $words = trim($line_array[$i-1].". ".$line.". ".$line_array[$i+1]);

        #$words = trim_annotation($words);

        foreach my $cited_num (@cited_nums)
        {
          $cited_num = trim($cited_num);
          if($cited_hash{$cited_num} eq undef)
          {
            my %cited_paper;
            $cited_paper{"num"} = $cited_num;
            #$cited_paper{"type"} = $cited_type;
            $cited_paper{"words"} = $words;

            $cited_hash{$cited_num} = \%cited_paper;
          }else{
            my $pcited_paper = $cited_hash{$cited_num};

###############################test ignore different types ##########################
#            if($$pcited_paper{"type"} !~/$cited_type/)
#            {
#              $$pcited_paper{"type"} .= ",".$cited_type;
#            }
################################test ignore different type############################
             #if($$pcited_paper{"type"} eq $cited_type)
             #{
                $$pcited_paper{"words"} .= " - ".$words;
             #}#if($$pcited_paper{"type"} eq $cited_type)
          }#if($cited_hash{$cited_num} eq undef)
        }#foreach my $cited_num (@cited_nums)
      }#while($line=~/\[(\d{1,3}|(?:\d{1,3},\s*)+\d{1,3})\]/g)
      last if($line =~/(References|LITERATURE CITED)\s*\[\d\]/i)
    }

    # parse references
    my $references;
    $content =~s/.*((?:References|LITERATURE CITED)\s*\[\d\].*)/$1/is;
    while($content=~s/\[(\d{1,3})\].*?<title>(.*?)<\/title>//s)
    {
      my $cited_num = $1;
      my $cited_title = $2;
      $cited_title = trim_content($cited_title);

###################################test ignore not downloaded papers########################
      next if ($download_papers{$cited_title} eq undef);
###################################test ignore not downloaded papers########################
      my $cited_index = $paper_index{$cited_title};
      if($cited_index eq undef)
      {
        $cited_index = scalar(@paper_array);
        $paper_index{$cited_title} = $cited_index;
        my %paper_hash;
        $paper_hash{"index"} = $cited_index;
        $paper_hash{"title"} = $cited_title;
        push @paper_array, \%paper_hash;
      }
      my $pcited_paper = $cited_hash{$cited_num};
      my $type = $$pcited_paper{"type"};
      my $words = $$pcited_paper{"words"};
      #if( ($cited_index ne undef) && ($words ne undef) && ($type ne undef))
      if( ($cited_index ne undef) && ($words ne undef))
      {
          $words =~s/(\[(\d+,\s*)*)$cited_num((,\s*\d+)*\])/$1\*\*\*$3/g;
          #$references .= $cited_index." ".$type." ".$words."|";
          $references .= $cited_index." 3 ".$words."|";
          $pair_number++;
      }
    }
    chop($references);
    my $ppaper = $paper_array[$index];
    $$ppaper{"references"} = $references;

  }
  print "Pair number:", $pair_number,"\n";

  # print precessed dataset into original file
  open outstream, ">".$outputfile;
  print outstream scalar(@paper_array),"\n";
  foreach my $ppaper(@paper_array)
  {
    print outstream "#@".$$ppaper{"index"},"\n";
    print outstream "#*".$$ppaper{"title"},"\n";
    print outstream "#&".$$ppaper{"references"},"\n";
    print outstream "#!".$$ppaper{"abstract"},"\n";
    print outstream "\n";
  }
  close outstream;
  $/ = $backup;

}

