package Bio::DB::GFF::Adaptor::memory;

=head1 NAME

Bio::DB::GFF::Adaptor::memory -- Bio::DB::GFF database adaptor for in-memory databases

=head1 SYNOPSIS

  use Bio::DB::GFF;
  my $db = Bio::DB::GFF->new(-adaptor=> 'memory',
                             -gff    => 'my_features.gff',
                             -fasta  => 'my_dna.fa'
                            );

or

  my $db = Bio::DB::GFF->new(-adaptor=>'memory');
  $db->load_gff_file('my_features.gff');
  $db->load_fasta_file('my_dna.fa');

See L<Bio::DB::GFF> for other methods.

=head1 DESCRIPTION

This adaptor implements an in-memory version of Bio::DB::GFF.  It can be used to
store and retrieve SHORT GFF files. It inherits from Bio::DB::GFF.

=head1 CONSTRUCTOR

Use Bio::DB::GFF-E<gt>new() to construct new instances of this class.
Three named arguments are recommended:

   Argument         Description

   -adaptor         Set to "memory" to create an instance of this class.
   -gff             Read the indicated file or directory of .gff file.
   -fasta           Read the indicated file or directory of fasta files.
   -dir             Indicates a directory containing .gff and .fa files

If you use the -dir option and the indicated directory is writable by
the current process, then this library will create a FASTA file index
that greatly diminishes the memory usage of this module.

Alternatively you may create an empty in-memory object using just the
-adaptor=E<gt>'memory' argument and then call the load_gff_file() and
load_fasta_file() methods to load GFF and/or sequence
information. This is recommended in CGI/mod_perl/fastCGI environments
because these methods do not modify STDIN, unlike the constructor.

=head1 METHODS

See L<Bio::DB::GFF> for inherited methods.

=head1 BUGS

none ;-)

=head1 SEE ALSO

L<Bio::DB::GFF>, L<bioperl>

=head1 AUTHOR

Shuly Avraham E<lt>avraham@cshl.orgE<gt>.

Copyright (c) 2002 Cold Spring Harbor Laboratory.

This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself.

=cut

use strict;
# $Id: memory.pm 15635 2009-04-14 19:11:13Z cjfields $
# AUTHOR: Shulamit Avraham
# This module needs to be cleaned up and documented

# Bio::DB::GFF::Adaptor::memory --  in-memory db adaptor
# implements the low level handling of data which stored in memory.
# This adaptor implements a specific in memory schema that is compatible with Bio::DB::GFF.
# Inherits from Bio::DB::GFF.


use Bio::DB::GFF::Util::Rearrange; # for rearrange()
use Bio::DB::GFF::Adaptor::memory::iterator;
use File::Basename 'dirname';
use Bio::DB::GFF::Adaptor::memory::feature_serializer qw(@hash2array_map);


use constant MAX_SEGMENT => 1_000_000_000;  # the largest a segment can get

use base qw(Bio::DB::GFF);

sub new {
  my $class = shift ;
  my ($file,$fasta,$dbdir,$preferred_groups) = rearrange([
							  [qw(GFF FILE)],
							  'FASTA',
							  [qw(DSN DB DIR DIRECTORY)],
							  'PREFERRED_GROUPS',
							 ],@_);

  # fill in object
  my $self = bless{ data => [] },$class;
  $self->preferred_groups($preferred_groups) if defined $preferred_groups;
  $file  ||= $dbdir;
  $fasta ||= $dbdir;
  $self->load_gff($file)             if $file;
  $self->load_or_store_fasta($fasta) if $fasta;
  return $self;
}

sub load_or_store_fasta {
  my $self  = shift;
  my $fasta = shift;
  if ((-f $fasta && -w dirname($fasta))
      or
      (-d $fasta && -w $fasta)) {
    require Bio::DB::Fasta;
    my $dna_db = eval {Bio::DB::Fasta->new($fasta);}
      or warn "$@\nCan't open sequence file(s). Use -gff instead of -dir if you wish to load features without sequence.\n";
    $dna_db && $self->dna_db($dna_db);
  } else {
    $self->load_fasta($fasta);
  }
}

sub dna_db {
  my $self = shift;
  my $d    = $self->{dna_db};
  $self->{dna_db} = shift if @_;
  $d;
}

sub insert_sequence {
  my $self = shift;
  my($id,$offset,$seq) = @_;
  $self->{dna}{$id} .= $seq;
}

# low-level fetch of a DNA substring given its
# name, class and the desired range.
sub get_dna {
  my $self = shift;
  my ($id,$start,$stop,$class) = @_;
  if (my $dna_db = $self->dna_db) {
    return $dna_db->seq($id,$start=>$stop);
  }
  return '' unless $self->{dna};

  return $self->{dna}{$id} unless defined $start || defined $stop;
  $start = 1 if !defined $start;

  my $reversed = 0;
  if ($start > $stop) {
    $reversed++;
    ($start,$stop) = ($stop,$start);
  }
  my $dna = substr($self->{dna}{$id},$start-1,$stop-$start+1);
  if ($reversed) {
    $dna =~ tr/gatcGATC/ctagCTAG/;
    $dna = reverse $dna;
  }

  $dna;
}

sub setup_load {
  my $self = shift;
  $self->{tmp}  = {};
  $self->{data} = [];
  1;
}

sub finish_load {
  my $self = shift;
  my $idx  = 0;
  foreach my $arrayref (values %{$self->{tmp}}) {
    foreach (@$arrayref) {$_->{feature_id} = $idx++; }
    push @{$self->{data}},@$arrayref;
  }
  1;
}

# this method loads the feature as a hash into memory -
# keeps an array of features-hashes as an in-memory db
sub load_gff_line {
  my $self = shift;
  my $feature_hash  = shift;
  $feature_hash->{strand} = ''  if $feature_hash->{strand} && $feature_hash->{strand} eq '.';
  $feature_hash->{phase}  = ''  if $feature_hash->{phase}  && $feature_hash->{phase} eq '.';
  $feature_hash->{gclass} = 'Sequence' unless length $feature_hash->{gclass} > 0;
  # sort by group please
  push @{$self->{tmp}{$feature_hash->{gclass},$feature_hash->{gname}}},$feature_hash;
}

# given sequence name, return (reference,start,stop,strand)
sub get_abscoords {
  my $self = shift;
  my ($name,$class,$refseq) = @_;
  my %refs;
  my $regexp;
  
  if ($name =~ /[*?]/) {  # uh oh regexp time
    $name = quotemeta($name);
    $name =~ s/\\\*/.*/g;
    $name =~ s/\\\?/.?/g;
    $regexp++;
  }

  # Find all features that have the requested name and class.
  # Sort them by reference point.
  for my $feature (@{$self->{data}}) {

    my $no_match_class_name;
    my $empty_class_name;
    my $class_matches = !defined($feature->{gclass}) ||
      length($feature->{gclass}) == 0 ||
	$feature->{gclass} eq $class;

    if (defined $feature->{gname}) {
      my $matches = $class_matches
	&& ($regexp ? $feature->{gname} =~ /$name/i : lc($feature->{gname}) eq lc($name));
      $no_match_class_name = !$matches;  # to accomodate Shuly's interesting logic
    }

    else{
      $empty_class_name = 1;
    }

    if ($no_match_class_name){
      my $feature_attributes = $feature->{attributes};
      my $attributes = {Alias => $name};
      if (!$self->_matching_attributes($feature_attributes,$attributes)){
	next;
      }
    }

    push @{$refs{$feature->{ref}}},$feature;
  }

  # find out how many reference points we recovered
  if (! %refs) {
    $self->error("$name not found in database");
    return;
  }

  # compute min and max
  my ($ref) = keys %refs;
  my @found = @{$refs{$ref}};
  my ($strand,$start,$stop);

  my @found_segments;
  foreach my $ref (keys %refs) {
    next if defined($refseq) and lc($ref) ne lc($refseq);
    my @found = @{$refs{$ref}};
    my ($strand,$start,$stop,$name);
    foreach (@found) {
      $strand ||= $_->{strand};
      $strand = '+' if $strand && $strand eq '.'; 
      $start  = $_->{start} if !defined($start) || $start > $_->{start};
      $stop   = $_->{stop}  if !defined($stop)  || $stop  < $_->{stop};
      $name ||= $_->{gname};
    }
    push @found_segments,[$ref,$class,$start,$stop,$strand,$name];

  }

  return \@found_segments;
}

sub search_notes {
  my $self = shift;
  my ($search_string,$limit) = @_;

  $search_string =~ tr/*?//d;

  my @results;
  my @words = map {quotemeta($_)} $search_string =~ /(\w+)/g;
  my $search = join '|',@words;

  for my $feature (@{$self->{data}}) {
    next unless defined $feature->{gclass} && defined $feature->{gname}; # ignore NULL objects
    next unless $feature->{attributes};
    my @attributes = @{$feature->{attributes}};
    my @values     = map {$_->[1]} @attributes;
    my $value      = "@values";
    my $matches    = 0;
    for my $w (@words) {
      my @hits = $value =~ /($w)/ig;
      $matches += @hits;
    }
    next unless $matches;

    my $relevance = 10 * $matches;
    my $featname = Bio::DB::GFF::Featname->new($feature->{gclass}=>$feature->{gname});
    my $note;
    $note    = join ' ',map {$_->[1]} grep {$_->[0] eq 'Note'}                @{$feature->{attributes}};
    $note   .= join ' ',grep /$search/,map {$_->[1]} grep {$_->[0] ne 'Note'} @{$feature->{attributes}};
    my $type = Bio::DB::GFF::Typename->new($feature->{method},$feature->{source});
    push @results,[$featname,$note,$relevance,$type];
    last if defined $limit && @results >= $limit;
  }

  #added result filtering so that this method returns the expected results
  #this section of code used to be in GBrowse's do_keyword_search method

  my $match_sub = 'sub {';
  foreach (split /\s+/,$search_string) {
    $match_sub .= "return unless \$_[0] =~ /\Q$_\E/i; ";
  }
  $match_sub .= "};";
  my $match = eval $match_sub;

  my @matches = grep { $match->($_->[1]) } @results;

  return @matches;
}

sub _delete_features {
  my $self        = shift;
  my @feature_ids = sort {$b<=>$a} @_;
  my $removed = 0;
  foreach (@feature_ids) {
    next unless $_ >= 0 && $_ < @{$self->{data}};
    $removed += defined splice(@{$self->{data}},$_,1);
  }
  $removed;
}

sub _delete {
  my $self = shift;
  my $delete_spec = shift;
  my $ranges      = $delete_spec->{segments} || [];
  my $types       = $delete_spec->{types}    || [];
  my $force       = $delete_spec->{force};
  my $range_type  = $delete_spec->{range_type};

  my $deleted = 0;
  if (@$ranges) {
    my @args = @$types ? (-type=>$types) : ();
    push @args,(-range_type => $range_type);
    my %ids_to_remove = map {$_->id => 1} map {$_->features(@args)} @$ranges;
    $deleted = $self->delete_features(keys %ids_to_remove);
  } elsif (@$types) {
    my %ids_to_remove = map {$_->id => 1} $self->features(-type=>$types);
    $deleted = $self->delete_features(keys %ids_to_remove);
  } else {
    $self->throw("This operation would delete all feature data and -force not specified")
      unless $force;
    $deleted = @{$self->{data}};
    @{$self->{data}} = ();
  }
  $deleted;
}

# attributes -

# Some GFF version 2 files use the groups column to store a series of
# attribute/value pairs.  In this interpretation of GFF, the first such
# pair is treated as the primary group for the feature; subsequent pairs
# are treated as attributes.  Two attributes have special meaning:
# "Note" is for backward compatibility and is used for unstructured text
# remarks.  "Alias" is considered as a synonym for the feature name.
# If no name is provided, then attributes() returns a flattened hash, of
# attribute=>value pairs.

sub do_attributes{
  my $self = shift;
  my ($feature_id,$tag) = @_;
  my $attr ;

  #my $feature = ${$self->{data}}[$feature_id];
  my $feature = $self->_basic_features_by_id($feature_id);

  my @result;
  for my $attr (@{$feature->{attributes}}) {
    my ($attr_name,$attr_value) = @$attr ;
    if (defined($tag) && lc($attr_name) eq lc($tag)){push @result,$attr_value;}
    elsif (!defined($tag)) {push @result,($attr_name,$attr_value);}
  }
  return @result;
}


#sub get_feature_by_attribute{
sub _feature_by_attribute{
  my $self = shift;
  my ($attributes,$callback) = @_;
  $callback || $self->throw('must provide a callback argument');
  my $count = 0;
  my $feature_id = -1;
  my $feature_group_id = undef;

  for my $feature (@{$self->{data}}) {

    $feature_id++;
    for my $attr (@{$feature->{attributes}}) {
      my ($attr_name,$attr_value) = @$attr ;
      #there could be more than one set of attributes......
      foreach (keys %$attributes) {
	if (lc($_) eq lc($attr_name) && lc($attributes->{$_}) eq lc($attr_value)) {
	  $callback->($self->_hash_to_array($feature));
	  $count++;
	}
      }
    }
  }

}


# This is the low-level method that is called to retrieve GFF lines from
# the database.  It is responsible for retrieving features that satisfy
# range and feature type criteria, and passing the GFF fields to a
# callback subroutine.

sub get_features{
  my $self = shift;
  my $count = 0;
  my ($search,$options,$callback) = @_;

  my $found_features;

  $found_features = $self->_get_features_by_search_options($search,$options);

  # only true if the sort by group option was specified
  @{$found_features} = sort {lc("$a->{gclass}:$a->{gname}") cmp lc("$b->{gclass}:$b->{gname}")}
    @{$found_features} if $options->{sort_by_group} ;

  for my $feature (@{$found_features}) {  # only true if the sort by group option was specified
    $count++;
    $callback->(
		$self->_hash_to_array($feature)
	       );
  }

  return $count;
}


# Low level implementation of fetching a named feature.
# GFF annotations are named using the group class and name fields.
# May return zero, one, or several Bio::DB::GFF::Feature objects.

=head2 _feature_by_name

 Title   : _feature_by_name
 Usage   : $db->get_features_by_name($name,$class,$callback)
 Function: get a list of features by name and class
 Returns : count of number of features retrieved
 Args    : name of feature, class of feature, and a callback
 Status  : protected

This method is used internally.  The callback arguments are those used
by make_feature().

=cut

sub _feature_by_name {
  my $self = shift;
  my ($class,$name,$location,$callback) = @_;
  $callback || $self->throw('must provide a callback argument');
  my $count = 0;
  my $regexp;

  if ($name =~ /[*?]/) {  # uh oh regexp time
    $name = quotemeta($name);
    $name =~ s/\\\*/.*/g;
    $name =~ s/\\\?/.?/g;
    $regexp++;
  }

  for my $feature (@{$self->{data}}) {
    next unless ($regexp && $feature->{gname} =~ /$name/i) || lc($feature->{gname})  eq lc($name);
    next if defined($feature->{gclass}) && length($feature->{gclass}) > 0 && $feature->{gclass} ne $class;

    if ($location) {
      next if $location->[0] ne $feature->{ref};
      next if $location->[1] && $location->[1] > $feature->{stop};
      next if $location->[2] && $location->[2] < $feature->{start};
    }
    $count++;
    $callback->($self->_hash_to_array($feature),0);
  }
  return $count;
}

# Low level implementation of fetching a feature by it's id. 
# The id of the feature as implemented in the in-memory db, is the location of the 
# feature in the features hash array.
sub _feature_by_id{
  my $self = shift;
  my ($ids,$type,$callback) = @_;
  $callback || $self->throw('must provide a callback argument');

  my $feature_group_id = undef;

  my $count = 0;
  if ($type eq 'feature'){
    for my $feature_id (@$ids){
      my $feature = $self->_basic_features_by_id($feature_id);
      $callback->($self->_hash_to_array($feature)) if $callback;
      $count++;
    }
  }
}

sub _basic_features_by_id{
  my $self = shift;
  my ($ids) = @_;
  
  $ids = [$ids] unless ref $ids =~ /ARRAY/;

  my @result;
  for my $feature_id (@$ids){
	  push @result, ${$self->{data}}[$feature_id];
  }
  return wantarray() ? @result : $result[0];
}

# This method is similar to get_features(), except that it returns an
# iterator across the query.
# See Bio::DB::GFF::Adaptor::memory::iterator.

sub get_features_iterator {
  my $self = shift;
  my ($search,$options,$callback) = @_;
  $callback || $self->throw('must provide a callback argument');

  my $results = $self->_get_features_by_search_options($search,$options);
  my $results_array = $self->_convert_feature_hash_to_array($results);

  return Bio::DB::GFF::Adaptor::memory::iterator->new($results_array,$callback);
}


# This method is responsible for fetching the list of feature type names.
# The query may be limited to a particular range, in
# which case the range is indicated by a landmark sequence name and
# class and its subrange, if any.  These arguments may be undef if it is
# desired to retrieve all feature types.

# If the count flag is false, the method returns a simple list of
# Bio::DB::GFF::Typename objects.  If $count is true, the method returns
# a list of $name=>$count pairs, where $count indicates the number of
# times this feature occurs in the range.

sub get_types {
  my $self = shift;
  my ($srcseq,$class,$start,$stop,$want_count,$typelist) = @_;

  my(%result,%obj);

  for my $feature (@{$self->{data}}) {
    my $feature_start = $feature->{start};
    my $feature_stop  = $feature->{stop};
    my $feature_ref   = $feature->{ref};
    my $feature_class = $feature->{class};
    my $feature_method = $feature->{method};
    my $feature_source = $feature->{source};

    if (defined $srcseq){
      next unless lc($feature_ref) eq lc($srcseq);
    }

    if (defined $class){ 
      next unless defined $feature_class && $feature_class eq $class ;
    }

     # the requested range should OVERLAP the retrieved features
     if (defined $start or defined $stop) {
      $start = 1           unless defined $start;
      $stop  = MAX_SEGMENT unless defined $stop;
      next unless $feature_stop >= $start && $feature_start <= $stop;
    }

    if (defined $typelist && @$typelist){
      next unless $self->_matching_typelist($feature_method,$feature_source,$typelist);
    }

    my $type = Bio::DB::GFF::Typename->new($feature_method,$feature_source);
    $result{$type}++;
    $obj{$type} = $type;

  }   #end features loop

  return $want_count ? %result : values %obj;
}

sub classes {
  my $self = shift;
  my %classes;
  for my $feature (@{$self->{data}}) {
    $classes{$feature->{gclass}}++;
  }
  my @classes = sort keys %classes;
  return @classes;
}

# Internal method that performs a search on the features array, 
# sequentialy retrieves the features, and performs a check on each feature
# according to the search options.
sub _get_features_by_search_options{
  my $count = 0;
  my ($self, $search,$options) = @_;
  my ($rangetype,$refseq,$class,$start,$stop,$types,$sparse,$order_by_group,$attributes) = 
    (@{$search}{qw(rangetype refseq refclass start stop types)},
     @{$options}{qw(sparse sort_by_group ATTRIBUTES)}) ;

  my @found_features;
  my $data = $self->{data};

  my $feature_id       = -1 ;
  my $feature_group_id = undef;

  for my $feature (@{$data}) {

    $feature_id++;

    my $feature_start = $feature->{start};
    my $feature_stop  = $feature->{stop};
    my $feature_ref   = $feature->{ref};

    if (defined $refseq){
      next unless lc($feature_ref) eq lc($refseq);
    }

     if (defined $start or defined $stop) {
      $start = 0               unless defined($start);
      $stop  = MAX_SEGMENT     unless defined($stop);

      if ($rangetype eq 'overlaps') {
	next unless $feature_stop >= $start && $feature_start <= $stop;
      } elsif ($rangetype eq 'contains') {
	next unless $feature_start >= $start && $feature_stop <= $stop;
      } elsif ($rangetype eq 'contained_in') {
	next unless $feature_start <= $start && $feature_stop >= $stop;
      } else {
	next unless $feature_start == $start && $feature_stop == $stop;
      }

    }

    my $feature_source = $feature->{source};
    my $feature_method = $feature->{method};

    if (defined $types && @$types){
      next unless $self->_matching_typelist($feature_method,$feature_source,$types);
    }

    my $feature_attributes = $feature->{attributes};
    if (defined $attributes){
      next unless $self->_matching_attributes($feature_attributes,$attributes);
    }

    # if we get here, then we have a feature that meets the criteria.
    # Then we just push onto an array
    # of found features and continue. 

    my $found_feature = $feature ;
    $found_feature->{feature_id} = $feature_id;
    $found_feature->{group_id} = $feature_group_id;
    push @found_features,$found_feature;
  }

  return \@found_features; 
}


sub _hash_to_array {
  my ($self,$feature_hash) = @_;
  my @array = @{$feature_hash}{@hash2array_map};
  return wantarray ? @array : \@array;
}

# this subroutine is needed for convertion of the feature from hash to array in order to 
# pass it to the callback subroutine
sub _convert_feature_hash_to_array{
  my ($self, $feature_hash_array) = @_;
  my @features_array_array = map {scalar $self->_hash_to_array($_)} @$feature_hash_array;
  return \@features_array_array;
}

sub _matching_typelist{ 
  my ($self, $feature_method,$feature_source,$typelist) = @_; 
  foreach (@$typelist) {
	 my ($search_method,$search_source) = @$_;
	 next if lc($search_method) ne lc($feature_method);
	 next if defined($search_source) && lc($search_source) ne lc($feature_source);
	 return 1;
  }
  return 0;
}

sub _matching_attributes {
  my ($self, $feature_attributes,$attributes) = @_ ;
  foreach (keys %$attributes) {
    return 0 if !_match_all_attr_in_feature($_,$attributes->{$_},$feature_attributes)
  }
  return 1;
}

sub _match_all_attr_in_feature{
  my ($attr_name,$attr_value,$feature_attributes) = @_;
  for my $attr (@$feature_attributes) {
      my ($feature_attr_name,$feature_attr_value) = @$attr ;
      next if ($attr_name ne $feature_attr_name || $attr_value ne $feature_attr_value);
      return 1;
  }
  return 0;
}


sub do_initialize { 1; }
sub get_feature_by_group_id{ 1; }

1;