ウェブページ取得と解析

perlによるウェブページ取得と解析

#!/usr/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use HTML::TreeBuilder;

print("---------------------------------------- script start\n");
#ページ取得
my $url = "http://www.yahoo.co.jp";
my $ua = LWP::UserAgent->new;
$ua->agent('Mozilla');
my $res = $ua->get($url);
$ua->parse_head(0);
my $content = $res->content;

#取得したpページを解析
my $tree = HTML::TreeBuilder->new;
$tree->parse($content);
$tree->eof();

for my $attr ( $tree->look_down("href", qr{http://} ) ){
    my $txt = $attr->as_text;
    if($txt ne ""){
        print $attr->as_text."\n";    
    }
}

$tree->delete;
print("---------------------------------------- script end\n");
exit();