Brian Duggan > Data-Downloader-0.9901 > Data::Downloader

Download:
Data-Downloader-0.9901.tar.gz

Dependencies

Annotate this POD

View/Report Bugs
Module Version: 0.9901   Source   Latest Release: Data-Downloader-0.9907

NAME ^

Data::Downloader -- Download and organize files using RSS feeds and templates

SYNOPSIS ^

    #!/usr/bin/env perl

    use Data::Downloader -init_logging => "DEBUG";
    use strict;

    # Get pictures of apples.  Store them using md5's, make symlinks using date and tags

    my $images = Data::Downloader::Repository->new(
        name           => "images",
        storage_root   => "/usr/local/datastore/data",
        cache_strategy => "Keep",
        feeds          => [
            {
                name => "flickr",
                feed_template => 'http://api.flickr.com/services/feeds/photos_public.gne?tags=<tags>&lang=en-us&format=rss_200',
                file_source => {
                    url_xpath      => 'media:content/@url',
                    filename_xpath => 'media:content/@url',
                    filename_regex => '/([^/]*)$',
                },
                metadata_sources => [
                    { name => 'date_taken', xpath => 'dc:date.Taken' },
                    { name => 'tags',       xpath => 'media:category' },
                ],
            },
        ],
        metadata_transformations => [
            {
                input         => "tags",
                output        => "tag",
                function_name => "split",
            },
        ],
        linktrees => [
            {
                root          => "/usr/local/datastore/by_tag",
                condition     => undef,
                path_template => "<tag>"
            },
            {
                root          => "/usr/local/datastore/by_date",
                condition     => undef,
                path_template => "<date_taken:%Y/%m/%d>"
            },
        ],
    );

    $images->load(speculative => 1) or $images->save;

    for my $feed ($images->feeds) {
        $feed->refresh(tags => "apples");
    }

    $images->download_all;

DESCRIPTION ^

Data::Downloader allows one to maintain local repositories of files and file metadata based on RSS feeds which describe the files. It stores files uniquely using MD5 sums, but allows for the creation and maintenance of trees of symbolic links to the downloaded files.

An SQLite database is used to store both the configuration and information about the files that have been downloaded. This file is automatically created in $HOME/.data_downloader.db (by default).

This distribution also conatins a command-line tool, dado, for managing the repository.

CONFIGURATION

See Data::Downloader::Config.

UPDATING RSS FEEDS

Read about this in Data::Downloader::Feed.

DOWNLOADING FILES

Take a look at Data::Downloader::File.

MAINTAINING SYMLINKS

Check out Data::Downloader::Linktree.

SEE ALSO ^

dado

Data::Downloader::Config

Data::Downloader::Repository

Data::Downloader::Feed

Data::Downloader::DB

Data::Downloader::Cache

Data::Downloader::Linktree

Rose

SCHEMA ^

__DATA__

-- See also Rose::DB::Object::Loader

 /*
  * Every table below corresponds to a class named Data::Downloader::<TableName>.
  * Every class has accessors and mutators whose names are the same as the column names.
  * Foreign keys also beget methods in both the parent and child classes, e.g.
  * e.g. $repository->files and $file->repository_obj.
  */

   /*** static tables : populated once during configuration ***/

--- A repository has a cache strategy.

    create table repository (
        id                integer primary key,
        name              varchar(255) not null unique,
        storage_root      text not null,
        file_url_template text,
        cache_strategy    text not null, -- e.g. "LRU" corresponds to Data::Downloader::Cache::LRU
        cache_max_size    integer -- in bytes
    );

--- A repository has many feeds.

    create table feed (
        id                integer primary key,
        repository        integer not null references repository(id),
        name              varchar(255) not null unique,
        feed_template     text
    );

--- A repository has many disks.

    create table disk (
        id                integer primary key,
        repository        integer not null references repository(id),
        root              varchar(255) not null unique
    );

--- A feed has many default parameters

    create table feed_parameter (
       id    integer primary key,
       feed  integer not null references feed(id),
       name  text not null,
       default_value text
    );

--- A feed has a file source.

    create table file_source (
        feed             integer primary key not null references feed(id),
        url_xpath        text,
        urn_xpath        text,  -- unique identifier for files
        md5_xpath        text,
        filename_xpath   text,
        filename_regex   text   -- apply to whatever is extracted from the xpath
    );

--- A feed has many metadata sources.

    create table metadata_source (
        id               integer primary key,
        feed             integer references feed(id),
        name             text not null unique,
        xpath            text
    );

--- A repository has many linktrees.

    create table linktree (
        id               integer primary key,
        repository       integer references repository(id),
        root             text,
        condition        text,      -- SQL::Abstract clause
        path_template    text,      -- String::Template string
        unique(root)
    );

--- A repository has many metadata_transformationss.

    create table metadata_transformation (
        id               integer primary key,
        input            text, -- references metadata_source.name or this table
        output           text not null,
        repository       integer references repository(id),
        function_name    text, -- function to apply
        function_params  text,
        order_key        integer not null default 1,
        unique(order_key)
    );

   /*** dynamic tables  : populated when files are downloaded, rss feeds are updated, etc. ***/

    create table stat_info (
        id integer primary key,
        repository integer not null references repository(id),
        last_stat_update datetime,
        last_fsck datetime,
        unique(repository)
    );

--- metadata

    create table metadatum (
        id     integer primary key,
        file   integer references file(id),
        name   varchar(255) references metadata_source(name),
        value  text,
        unique (file,name)
    );

--- A repository has many files.

    create table file (
       id                 integer primary key,
       repository         integer not null references repository(id),
       filename           text not null,
       md5                char(32),
       url                text,
       urn                text,
       size               integer,
       on_disk            integer, -- boolean
       disk               integer references disk(id),
       atime              datetime, -- stat(file)->atime
       unique(md5),
       unique(filename),
       unique(urn)
    );

--- A file has many log entries --- (if $ENV{DATA_DOWNLOADER_GATHER_STATS} is set).

    create table log_entry (
       id integer primary key,
       file integer not null references file(id),
       requested_at datetime not null,
       cache_hit integer, -- boolean
       completed_at datetime,
       prog text, -- $0
       pid integer, -- $$
       uid text, -- $<
       note text -- $ENV{DATA_DOWNLOADER_LOG_NOTE}, e.g. app info
    );

--- A file has many expirations

    create table file_removal (
       id integer primary key,
       file integer not null references file(id),
       expired_on datetime,
       algorithm, -- the expiration algorithm
       prog text, -- $0
       pid integer, -- $$
       uid text, -- $<
       note text -- $ENV{DATA_DOWNLOADER_FILE_REMOVAL_NOTE},
    );

--- A file has many symlinks.

    create table symlink (
       id                 integer primary key,
       linkname           text not null, -- linktree.path_template + file's metadata
       file               integer not null references file(id),
       linktree           integer not null references linktree(id),
       unique(linkname)
    );

/***\

AUTHOR ^

 Brian Duggan

 Phillip Durbin

 Stuart Pineo

 Arnold Martin

 Graham Ollis

 Michael Walters
syntax highlighting: