zhangbo-NLP-plugin_eng/lib/zhangbo/NLP/engine_rea_zhonto.pl
#!/usr/bin/perl
#use JSON::XS;
use Clone qw(clone);
use MongoDB;
use Smart::Comments;
use lib "/home/wuyabo/shell/";
use Conn_mongo_jc;
use Data::Dumper;
use IO::File;
#use Add_info;
open(FDst,"stop.txt");
our $hash_st;
while(<FDst>)
{
my $line;
$line=$_;
chomp($line);
$hash_st->{$line}=1;
}
# http://poe.perl.org/?POE_Cookbook/TCP_Servers
# Include POE and POE::Component::Server::TCP.
our $hash=();
$hash_mn=();
my $hash_sc=();
#i#close FD_log;
my $w1=$ARGV[0];
my $w2=$ARGV[1];
my $f1=$ARGV[2];
my $f2=$ARGV[3];
my $t=search_article($f1);
$res=clone($t);
# print Data::Dumper->Dump([$res]);
my $t=search_article($f2);
$res2=clone($t);
my $k,$v;
my $tv1;
my $tv2;
while(($k,$v)=each(%{$res}))
{
if(($k eq $w1)||($k eq $w2))
{next;}
if(exists($res2->{$k}))
{
$res3->{$k}=($res2->{$k})+($res->{$k});
# print "---same : $k $res3->{$k}\n ";
}
}
my @key =sort {$res3->{$b} <=> $res3->{$a}} keys %{$res3};
my @re;
foreach my $ttk (@key)
{
print "$ttk-$res3->{$ttk}\n";
#system ("grep $ttk $f1");
}
#电影-25.02-
#美国-50.75-
#纳多-21.26-
#print FD_log @res;
#close(FD_log);
#print scalar(@res)," num\n";
#$session->yield("shutdown");
#, ClientFlushed => sub {
# my $data_source = $_[HEAP]{file_handle};
# my $read_count = sysread($data_source, my $buffer = "", 65536);
# if ($read_count) {
# $_[HEAP]{client}->put($buffer);
# }
# else {
# print FD_log "------------------error\n";
# $_[KERNEL]->yield("shutdown");
# }
#}
#$input="坦克,我的希望非常诱人电视里有主持人，那里有大熊猫";
# $input="网络和网民的意见，现在正行驶在通州回北京的高速公路上，十评论员单仁平的文章，标题是“做大众政治焦点，茅于轼的选择”。这篇文章的核心意见是，茅于轼应该做中国社会团结的促进者，不应该";
#$input="致力于宣传 市场万能 剥削有理 汉奸人性 保钓无用 保粮错误替富人说话的茅于轼，今天下午两点在北京海淀翠宫饭店演讲顽强继续。未知海淀区委书记隋振江，宣传部长陈名杰是否到场。外媒问我是否到场，告曰:先参加央视《苦难辉煌》座谈会，或会晚到一会儿。今天要长见识了";
# $input2="茅于轼刚刚吃完胡辣汤和烧饼夹猪头肉。现在正行驶在通州回北京的高速公路上，十五分钟以后要开始腾讯微访谈直播，我和方舟子拟就昨天的热点问题，回答网友的提问。敬请各位网友提示一下，昨天有什么热点问题需要谈一谈？";
#$file=$ARGV[0];
sub search_article
{
my $file=$_[0];
chomp($file);
if(! -e "$file")
{return "file is't exist\n"};
my %my_hash;
my %my_hash2;
my %thash;
my $j;
my $rand=rand();
#open(FD_hash,">$file.txt");
#print "openfile $file\n";
system("/home/wuyabo/xs/bin/scws -I -t 500 -E -N -d /home/lzj/shell/dict.xdb -c utf-8 $file -o /tmp/$rand.sctxt2");
my @all2=`cat /tmp/$rand.sctxt2`;
#20. 建中 nz 13.84(1)
#21. 恒大 nz 13.41(1)
unlink("/tmp/$rand.sctxt2");
my $hash3;
foreach $j (@all2)
{
#print "$j\n";
my $word;
my $kv;
my $pos;
if($j=~/\d(\d)\. (.*?)\s+(\w+?)\s+(\d.*?)\(/)
{
$word=$2;
$kv=$4;
$pos=$3;
if(($pos=~/n.*/)||($pos=~/v.*/))
{
$hash3->{$word}=$kv;
}
}
}
$hash3;
}
# }
# );
#$poe_kernel->run();
sub txt2arr
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
# chomp($input_t);
my @list=split(" ",$input_t);
#print "txt2arr input =$input_t\n";
#$rand=rand();
#open(FD_r,">/tmp/$rand");
#print FD_log "$input_t\n";
#close(FD_r);
#$dir="/home/wuyabo/stanford-parser-2012-07-09";
@res=();
#@res=`$dir/parser-10.pl /tmp/$rand 0 50`;
#$part=0;
#unlink("/tmp/$rand");
#print @res;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
if(exists($hash_st->{$tmp_res}))
{next;}
my $l=length($tmp_res);
# if($l<=3)
# {
#if($hash_sc->{$tmp_res}->{'a'} !~/n|v|a/)
#{
#print FD_log "seek $tmp_res\n";
#next;}
if($l<=3)
{next;}
#主题放大
if(exists($hash_sc->{$tmp_res}))
{
if($hash_sc->{$tmp_res}->{'a'}=~/n/)
{
if($hash_sc->{$tmp_res}->{'h'}> $n_a)
{
$n_a=$hash_sc->{$tmp_res}->{'h'};
$n1=$tmp_res;
# print FD_log "放大主题-$tmp_res-","$n_a n\n";
if($n_a >$n_b)
{
my $swtmpn=$n_b;
my $swtmpnw=$n2;
$n_b=$n_a;$n2=$n1;
$n_a=$swtmpn;$n1=$swtmpnw;
}
}
}
elsif($hash_sc->{$tmp_res}->{'a'}=~/v/)
{
if($hash_sc->{$tmp_res}->{'h'}> $v_a)
{
$v_a=$hash_sc->{$tmp_res}->{'h'};
$v1=$tmp_res;
print FD_log "放大主题-$tmp_res-","$v_a v \n";
if($v_a >$v_b)
{
my $swtmpn=$v_b;
my $swtmpnw=$v2;
$v_b=$v_a;$v2=$v1;
$v_a=$swtmpn;$v1=$swtmpnw;
}
}
}
}
# }
# }
# print $tmp_res,"---",length($tmp_res),"\n";
#if (($part==1)&&($tmp_res=~/(.*?),(.*)/))
#{
#print "切词1=$1,2=$2\n";
my $w=$tmp_res;
# my $value=$2;
# if(exists($exp_h->{$w}))
# {
# print "$w\n";
# next;}
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each($hash->{$w}))
{
if(exists($hash_st->{$k}))
{next;}
$tmp_hash->{$k}+=0.1;
#$tmp_hash->{$k}=$v;
#$tmp_hash->{$k}=$hash_sc->{$w}->{'h'};
#print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
#正常词 交 转换0.00--0.00 相互
#if(exists($sytmp->{$k}))
#{
# print "归一 $k ",$sytmp->{$k},"\n";
# $k=$sytmp->{$k};
#}
# if($value>20)
# $heap->{client}->put("$k $v\n");
}
#$tmp_hash->{$k}=$v;
}
else
{
if(exists($hash_sc->{$w}))
{
$tmp_hash->{$w}+=0.2;
#$tmp_hash->{$w}=$hash_sc->{$w}->{'h'};
}
else
{
$tmp_hash->{$w}=0.1;
}
# print FD_log "no exists wordnet 切词 k=$w,v=",$hash_sc->{$w}->{'h'},"\n";
}
}
#是否采用scws放大文章中心词
while(($k,$v)=each(%{$tmp_hash}))
{
if(($k eq $n1)||($k eq $n2)||($k eq $v1)||($k eq $v2))
{$tmp_hash->{$k}=$v*2;
}
if(exists($exp_h->{$k}))
{
$tmp_hash->{$k}+=0.8;
# print "自定义放大主题-$k-",$tmp_hash->{$k}," \n";
}
}
#print Data::Dumper->Dump([%tmp_hash]);
#return (%tmp_hash,$nline);
return (\%{$tmp_hash},$nline);
}
#print "output = $res\n";
# $heap->{client}->stop();
# $session->stop();
# Start the server.
sub comp_hash
{
my $h1=@_[0];
my $h2=@_[1];
my $akv=0;
my $all;
my %all;
my $vall=0.01;
my $vall2=0.01;
my $k,$v;
while(($k,$v)=each(%{$h1}))
{
$all->{$k}++;
$vall+=$v;
# print FD_log " hash 1$k $v\n";
}
my $k,$v;
while(($k,$v)=each(%{$h2}))
{
$all->{$k}++;
# print FD_log " hash 2$k $v\n";
$vall2+=$v;
}
my $k,$v;
while(($k,$v)=each(%{$all}))
{
if($v >= 2)
{
$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
$kv2=((($h1->{$k})/$vall) + ($h2->{$k}/$vall2))/2;
$kv3=($kv+$kv2)/2;
#if($h1->{$k}<$h2->{$k})
#{
#}
# print FD_log "double $k $v kv $kv kv2 $kv2\n";
$akv+=$kv3;
}
}
# print "---------------\n";
$akv;
}
sub ndate
{
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time());
$year=~s/1(.*)/20$1/;
$mon+=1;
if($mon=~/^\d$/)
{
$mon="0"."$mon";
}
if($mday=~/^\d$/)
{
$mday="0"."$mday";
}
if($hour=~/^\d$/)
{
$hour="0"."$hour";
}
if($min=~/^\d$/)
{
$min="0"."$min";
}
if($sec=~/^\d$/)
{
$sec="0"."$sec";
}
my $res=$year."-".$mon."-".$mday." ".$hour.":"."$min".":"."$sec";
#print $res,"\n";
$res;
}
sub txt2arr_one
{
my $w=$_[0];
my $exp_h=$_[1];
$tmp_hash=();
$rand=rand();
@res=();
$part=0;
#print @res;
my @w_arr=split(",",$w);
foreach my $w_tmp (@w_arr)
{
if($w_tmp=~/(.*?) (.*)/)
{
$w=$1;
$vh=$2;
#print "$w -- $vh\n";
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each($hash->{$w}))
{
if($v==0)
{
$v=1;
}
$tmp_hash->{$k}=$vh;
}
}
$tmp_hash->{$w}=$w;
}
else
{
print "input error\n";
}
}
# while(($k,$v)=each(%{$tmp_hash}))
#
# {
#print "meaning $k $v\n";
# }
# }
return (%{$tmp_hash});
}
sub search_article2
{
$file=$_[0];
chomp($file);
#print "file=$file\n";
$expword=$ARGV[1];
$exp_hash=();
@exp_w=split(",",$expword);
foreach $exp (@exp_w)
{
$exp_hash->{$exp}=1;
}
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %thash;
my $j;
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
while(<FD_arr>)
{
my $in=$_;
chomp($in);
print FD_aro "$j ",$in,"\n";
$j++;
}
close FD_arr;
close FD_aro;
system("/home/wuyabo/xs/bin/scws -I -E -N -d /home/wuyabo/xs/etc/dict.utf8.xdb:/home/wuyabo/stanford/dict_user.txt -c utf-8 /tmp/$rand.sctxt -o /tmp/$rand.sctxt2");
unlink("/tmp/$rand.sctxt");
#print "/tmp/$rand.sctxt\n";
open(FD_scws,"/tmp/$rand.sctxt2");
my $hashleft;
my $alln=1;
while(<FD_scws>)
{
my $in=$_;
#print FD_log "line ttt $kk $in \n";
chomp($in);
#$input_ar[$f]=$in;
my $n_line;
my ($my_hash2,$n_line)=txt2arr($in,$exp_hash);
$hashleft[$n_line]=clone($my_hash2);
$alln++;
}
my $j=0;
my $cluster,$cln;
for( 1 .. $alln-1)
{
my $nn=$_;
for ($nn .. $alln-1)
{
$nn2=$_;
if($nn==$nn2)
{next;}
$n=comp_hash($hashleft[$nn],$hashleft[$nn2]);
$n=$n * 100;
# print "$n $nn $nn2 \n";
$cluster->{"$nn-$nn2"}=$n;
$cln[$j]="$nn-$nn2";$j++;
# print FD_log "$n $nn $nn2 \n";
}
}
my @m_res;
my @key =sort {$cluster->{$b} <=> $cluster->{$a}} keys %{$cluster};
for(0 .. (scalar(@key))-1)
{
my $itmp=$_;
# print FD_log $key[$itmp],"-",$cluster->{$key[$itmp]},"\n";
push @m_res,"$key[$itmp]-$cluster->{$key[$itmp]}\n";
}
unlink("/tmp/$rand.sctxt2");
close(FD_arr);
@m_res;
}