OSSP CVS Repository

ossp-pkg/quos/openpkg-rdf2sql.pl 1.7

---

#!/usr/lpkg/bin/perl
##
##  OSSP quos - Query On Steroids
##  Copyright (c) 2004 Ralf S. Engelschall <rse@engelschall.com>
##  Copyright (c) 2004 The OSSP Project <http://www.ossp.org/>
##
##  This file is part of OSSP quos, a Web user interface for querying
##  a database which can be found at http://www.ossp.org/pkg/tool/quos/.
##
##  This program is free software; you can redistribute it and/or modify
##  it under the terms of the GNU General Public License as published by
##  the Free Software Foundation; either version 2 of the License, or
##  (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
##  General Public License for more details.
##
##  You should have received a copy of the GNU General Public License
##  along with this program; if not, write to the Free Software
##  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
##  USA, or contact Ralf S. Engelschall <rse@engelschall.com>.
##
##  openpkg-rdf2sql.cfg: OpenPKG XML/RDF to SQL importing tool
##

require 5.008;
use strict;
use Getopt::Long;
use XML::Parser;
use DBI;
use DBD::SQLite;
use Data::Dumper;
use XML::Simple;
use LWP::UserAgent;
use IPC::Filter qw();

#   configure optional debugging
$Data::Dumper::Purity = 1;
$Data::Dumper::Indent = 1;
$Data::Dumper::Terse  = 1;

my $ftpserv  = 'ftp://anonymous@ftp.openpkg.org/';
my $starturl = '00INDEX.rdf';

#   connect to database
my $db = DBI->connect("dbi:SQLite:dbname=openpkg.db", "", "",
                      { RaiseError => 1, AutoCommit => 1 });

#   prepare SQL commands
my $sql = {};
$sql->{-rdf} = $db->prepare(
    "INSERT INTO quos_rdf (rd_url) VALUES (?);"
);
$sql->{-package} = $db->prepare(
    "INSERT INTO quos_package" .
    " (pk_name, pk_version, pk_release, pk_distribution, pk_group," .
    "  pk_license, pk_packager, pk_summary, pk_url, pk_vendor, pk_description, pk_rdf)" .
    " VALUES (?,?,?,?,?,?,?,?,?,?,?,(SELECT MAX(rd_id) FROM quos_rdf));\n"
);
$sql->{-buildprereq} = $db->prepare(
    "INSERT INTO quos_buildprereq" .
    " (bp_id, bp_key, bp_op, bp_val)" .
    " VALUES ((SELECT MAX(pk_id) FROM quos_package),?,?,?);" 
);
$sql->{-prereq} = $db->prepare(
    "INSERT INTO quos_prereq" .
    " (rp_id, rp_key, rp_op, rp_val)" .
    " VALUES ((SELECT MAX(pk_id) FROM quos_package),?,?,?);" 
);
$sql->{-provide} = $db->prepare(
    "INSERT INTO quos_provide" .
    " (pr_id, pr_key, pr_op, pr_val)" .
    " VALUES ((SELECT MAX(pk_id) FROM quos_package),?,?,?);" 
);
$sql->{-source} = $db->prepare(
    "INSERT INTO quos_source" .
    " (sr_id, sr_url)" .
    " VALUES ((SELECT MAX(pk_id) FROM quos_package),?);" 
);

&fetchrdfsrecursendump2db($starturl);

sub fetchrdfsrecursendump2db {
    my ($url) = @_;

    #   fetch XML/RDF index file
    my $ua = new LWP::UserAgent;
    $ua->agent("openpkg-rdf2sql/1.0");
    my $req = new HTTP::Request(GET => $ftpserv . $url);
    my $rescont = '';
    my $res = $ua->request($req);
    if ($res->is_success) {
        if ($url =~ m|.*bz2$|) {
            $rescont = IPC::Filter::filter($res->content, "bzip2 -d");
        }
        else {
            $rescont = $res->content;
        }
    }
    else {
        print $res->status_line . "\n";
    }
    
    #   parse XML/RDF index file
    my $xml = new XML::Simple;
    my $rdf = $xml->XMLin(
        $rescont,
        KeepRoot     => 1,
        ForceContent => 0,
        ForceArray   => 1,
    );          
    undef $xml;

    #   iterate over XML/RDF data structure
    foreach my $repo (@{$rdf->{'rdf:RDF'}->[0]->{'Repository'}}) {
        my $rd_url = $repo->{'rdf:resource'};

        #   start SQL transaction
        $db->begin_work();
    
        #   store repository information
        $sql->{-rdf}->execute($rd_url);
    
        #   iterate over all packages in a repository
        if (defined($repo->{'rdf:Description'})) {
            foreach my $desc (@{$repo->{'rdf:Description'}}) {
                #   store simple (single-value) properties of a package
                my $prop = {};
                foreach my $attr (qw(
                    Name Version Release Distribution Group License
                    Packager Summary URL Vendor Description
                )) {
                    $prop->{$attr} = $desc->{$attr}->[0];
                }
                $sql->{-package}->execute(
                    $prop->{'Name'}, $prop->{'Version'}, $prop->{'Release'}, $prop->{'Distribution'}, $prop->{'Group'},
                    $prop->{'License'}, $prop->{'Packager'}, $prop->{'Summary'}, $prop->{'URL'}, $prop->{'Vendor'},
                    $prop->{'Description'}
                );
    
                #   store complex (multi-value) properties of a package
                foreach my $attr (qw(
                    BuildPreReq
                    PreReq
                    Provide
                )) {
                    foreach my $el (@{$desc->{$attr}->[0]->{'rdf:bag'}->[0]->{'resource'}}) {
                        my ($key, $op, $val) = ($el, '=', '*');
                        if (ref($key) eq 'HASH') {
                            $key = $el->{'content'};
                            $op  = (grep { $_ ne 'content' } keys(%{$el}))[0];
                            $val = $el->{$op};
                        }
                        $sql->{"-".lc($attr)}->execute($key, $op, $val);
                    }
                }
                foreach my $url (@{$desc->{'Source'}->[0]->{'rdf:bag'}->[0]->{'rdf:li'}}) {
                    $sql->{-source}->execute($url);
                }
            }
        }

        #   iterate over all sub-repositities in repository
        if (defined($repo->{'Repository'})) {
            foreach my $repcont (@{$repo->{'Repository'}}) {
                $url =~ m|^(.*/)|;
                my $actpath = $1;
                &fetchrdfsrecursendump2db($actpath . $repcont->{'href'}); 
                sleep(1);
            }
        }

        #   end SQL transaction
        $db->commit();
    }
}

#   commit and disconnect from database
$db->disconnect();

#   temporary function for showing actual XML
sub showactxml {
    my ($xmlinput) = @_;
    
    my $xml = new XML::Simple;
    my $rdf = $xml->XMLin(
        $xmlinput,
        KeepRoot     => 1,
        ForceContent => 0,
        ForceArray   => 1,
    );          
    undef $xml;
    print Dumper($rdf);
}

__END__

=pod

=head1 NAME

B<openpkg-rdf2sql> - OpenPKG XML/RDF to SQL importing tool

=head1 SYNOPSIS

B<openpkg-rdf2sql>
[B<-v>|B<--verbose>]
...FIXME...
I<url>

=head1 DESCRIPTION

=cut

---