zhangbo-NLP-plugin_eng/lib/zhangbo/NLP/em2.pl
#!/usr/bin/perl
#use JSON::XS;
use Clone qw(clone);
use MongoDB;
use Smart::Comments;
use lib "/home/wuyabo/shell/";
use Conn_mongo_jc;
use Data::Dumper;
use IO::File;
#use Add_info;
open(FDst,"stop.txt");
our $hash_st;
while(<FDst>)
{
my $line;
$line=$_;
chomp($line);
$hash_st->{$line}=1;
}
# http://poe.perl.org/?POE_Cookbook/TCP_Servers
# Include POE and POE::Component::Server::TCP.
our $hash=();
$hash_mn=();
my $hash_sc=();
open(FD_sc, "dict.utf8.txt");
open(FD_log, ">/tmp/engine.txt");
#while(<FD_sc>)
#{
#招收学员 13.87 8.50 n
# my $line=$_;
# if($line=~/(.*?)\t(.*?)\t(.*?)\t(.*)/)
# {
# $tt1=$1;
# $tt2=$2;
# $tt3=$3;
# $tt4=$4;
# $hash_sc->{$tt1}->{"i"}=$tt2;
# $hash_sc->{$tt1}->{"h"}=$tt3;
# $hash_sc->{$tt1}->{"a"}=$tt4;
#
# }
#}
open(FD, "word.txt");
my $tmp=();
while(<FD>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
chomp($line);
if($line=~/^W_C=(.*)/)
{
$tmp=$1;
#print $tmp,"\n";
$r=1;
}
elsif($line=~/^DEF=(.*)/)
{
$r=0;
$tmp_m=$1;
# print $tmp_m,"\n";
my @arry=split('\|',$tmp_m);
foreach $tmps (@arry)
{
# print "-----------------$tmps ------------\n";
if($tmps=~/(.*?)\:/)
{
my $m=$1;
if(exists($hash_st->{$m}))
{next;}
# if($m eq "专")
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
# print "$tmp - $m-----\n";
if(exists($hash_sc->{$m}))
{
if($r==0)
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
}
else{ $hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*0.2;}
#$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.1;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
$r++;
}
elsif($tmps=~/(.*?)\}/)
{
my $m=$1;
# if($m eq "专")
#if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件"))
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
if(exists($hash_st->{$m}))
{next;}
if($r==0)
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
}
else
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*(0.5);
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=0.5;
}
$r++;
}
}
}
}
open(FDb, "babel_all_path_fin20.txt");
my $tmp=();
while(<FDb>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
$s=$line;
# $m=$2;
my $name;
@a=split("\t",$s);
#@b=split(" ",$m);
$f=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$name=$i;
if(exists($hash->{$name}))
{
# print "$name\n";
last;
}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
if(exists($hash->{$name}->{$av}))
{next;}
$hash->{$name}->{$av}=$bv;
}
}
}
#foreach $i (@b)
#{
# $hash->{$name}->{$i}+=0.01;
#}
}
close FD_log;
print "server start\n";
use POE qw(Component::Server::TCP);
my $n=0;
#our $f=0;
POE::Component::Server::TCP->new(
Alias => "echo_server",
Port => 11221,
ClientInput => sub {
my ($session, $heap, $input) = @_[SESSION, HEAP, ARG0];
#print "Session ", $session->ID(), " got input: ".$input."\n";
#print "Session ", $session->ID(), " got input: ".length($input)."\n";
my $d;
open(FD_log,">/tmp/engine_mu.log");
print FD_log "$input\n";
#system("cp $input /tmp/");
print "input file $input",length($input),"\n";
my @res=();
my $res;
if ($input=~/^cluster/)
{
@res=search_article2($input);
}
else
{
@res=search_article($input);
}
print FD_log @res;
close(FD_log);
#print scalar(@res)," num\n";
$heap->{client}->put(@res);
#$session->yield("shutdown");
$_[KERNEL]->yield("shutdown");
return;
},
ClientDisconnected => sub {
#print "Client disconnected\n"; # log it
}
#, ClientFlushed => sub {
# my $data_source = $_[HEAP]{file_handle};
# my $read_count = sysread($data_source, my $buffer = "", 65536);
# if ($read_count) {
# $_[HEAP]{client}->put($buffer);
# }
# else {
# print FD_log "------------------error\n";
# $_[KERNEL]->yield("shutdown");
# }
#}
);
POE::Kernel->run;
#$input="坦克,我的希望非常诱人电视里有主持人，那里有大熊猫";
# $input="网络和网民的意见，现在正行驶在通州回北京的高速公路上，十评论员单仁平的文章，标题是“做大众政治焦点，茅于轼的选择”。这篇文章的核心意见是，茅于轼应该做中国社会团结的促进者，不应该";
#$input="致力于宣传 市场万能 剥削有理 汉奸人性 保钓无用 保粮错误替富人说话的茅于轼，今天下午两点在北京海淀翠宫饭店演讲顽强继续。未知海淀区委书记隋振江，宣传部长陈名杰是否到场。外媒问我是否到场，告曰:先参加央视《苦难辉煌》座谈会，或会晚到一会儿。今天要长见识了";
# $input2="茅于轼刚刚吃完胡辣汤和烧饼夹猪头肉。现在正行驶在通州回北京的高速公路上，十五分钟以后要开始腾讯微访谈直播，我和方舟子拟就昨天的热点问题，回答网友的提问。敬请各位网友提示一下，昨天有什么热点问题需要谈一谈？";
#$file=$ARGV[0];
sub search_article
{
my $file=$_[0];
chomp($file);
#print "file=$file\n";
$expword=$ARGV[1];
$exp_hash=();
@exp_w=split(",",$expword);
foreach $exp (@exp_w)
{
$exp_hash->{$exp}=1;
}
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash=();
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %thash;
my $j;
#my $rand2=rand();
#open(FD_hash,">$file.txt");
#print "openfile $file\n";
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
my @inputa;
my @inputb;
while(<FD_arr>)
{
#print FD_log $j,"----------\n";
#print $j,"----------\n";
my $in=$_;
if($first==0)
{
if($in=~/^-/)
{
$first++;
print FD_aro "$in";
}
else{
print FD_aro "0 $in";
push (@inputa,$in);
}
}
else
{
#if(length($in)>500)
#{ $in=substr $in,0,500;}
# $in=~s/\@.*?\:/ /g;
# $in=~s/\@.*?\s/ /g;
# $in=~s/机场|航班|深圳/ /g;
print FD_aro "$j $in";
push (@inputb,$in);
$j++;
#print FD_log $j,"-p----",length($in),"\n";
#print $j,"-p----",length($in),"\n";
}
}
close FD_arr;
close FD_aro;
system("/home/lzj/shell/jieba-0.31/test/test_file.py /tmp/$rand.sctxt /tmp/$rand.sctxt2 2>/dev/null");
#system("/home/wuyabo/xs/bin/scws -I -E -N -d /home/wuyabo/xs/etc/dict.utf8.xdb:/home/wuyabo/stanford/dict_user.txt -c utf-8 /tmp/$rand.sctxt -o /tmp/$rand.sctxt2");
system("cp /tmp/$rand.sctxt2 66.txt");
system("cp /tmp/$rand.sctxt 55.txt");
system("cp $file 44.txt");
unlink("/tmp/$rand.sctxt");
open(FD_scws,"</tmp/$rand.sctxt2");
print FD_log "open /tmp/$rand.sctxt2 \n";
my $disthash;
my @disthash;
while(<FD_scws>)
#foreach my $in (@output)
{
my $in=$_;
print FD_log "start \n";
print FD_log "line ttt $kk $in \n";
chomp($in);
#$input_ar[$f]=$in;
if($f==0)
{
if($in=~/^ - /)
{
$f++;
next;
}
#if(length($in)>12)
#{
my ($d,$c)=txt2arr3($in,$exp_hash);
$disthash[$kk]=clone($d);
print FD_log "----$kk\n";
$kk++;
#}
next;
#print Data::Dumper->Dump([%{$d}]);
# %my_hash2=txt2arr($input2);
# $n=comp_hash(\%my_hash,\%my_hash2);
}
else
{
my $n_line;
my ($my_hash2,$n_line)=txt2arr3($in,$exp_hash);
# while (($k,$v)=each(%my_hash2))
# {
# print FD_hash "$k,$v\t";
# }
# print FD_hash "\n";
for (0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
$n=comp_hash($disthash[$nnn],$my_hash2);
my $shnn=$nnn+1;
print FD_log "totle $shnn-$n_line=$n\n";
print FD_log "$inputa[$nnn]\n$inputb[$n_line-1]\n";
$thash[$nnn]->{$n_line}=$n;
}
}
$f++;
}
#unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
# close(FD_arr);
my $m_res;
my @m_res=();
for(0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
my @key =sort {$thash[$nnn]->{$b} <=> $thash[$nnn]->{$a}} keys %{$thash[$nnn]};
print FD_log "key---",@key,"\n";
my $o_n;
foreach $i (@key)
{
# if( $thash[$nnn]->{$i}>0)
# {
my $shnnn=$nnn+1;
if($thash[$nnn]->{$i} >0)
{
push (@m_res,"$shnnn-$i-".$thash[$nnn]->{$i});
print FD_log "add new---",$thash[$nnn]->{$i},"\n";
}
#push (@m_res,"$nnn-$i-".$thash[$nnn]->{$i}.$input_ar[$i]);
# $o_n++;
# }
}
}
print FD_log "return ",scalar(@m_res)," @m_res\n";
@m_res;
}
# }
# );
#$poe_kernel->run();
sub txt2arr
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
# chomp($input_t);
my @list=split(" ",$input_t);
#print "txt2arr input =$input_t\n";
#$rand=rand();
#open(FD_r,">/tmp/$rand");
print FD_log "$input_t\n";
#close(FD_r);
#$dir="/home/wyb/stanford-parser-2012-07-09";
@res=();
#@res=`$dir/parser-10.pl /tmp/$rand 0 50`;
#$part=0;
#unlink("/tmp/$rand");
#print @res;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
if(exists($hash_st->{$tmp_res}))
{next;}
my $l=length($tmp_res);
# if($l<=3)
# {
#if($hash_sc->{$tmp_res}->{'a'} !~/n|v|a/)
#{
#print FD_log "seek $tmp_res\n";
#next;}
if($l<=3)
{next;}
#主题放大
$tmp_hash->{$tmp_res}++;
# }
# print $tmp_res,"---",length($tmp_res),"\n";
#if (($part==1)&&($tmp_res=~/(.*?),(.*)/))
#{
#print "切词1=$1,2=$2\n";
my $w=$tmp_res;
# my $value=$2;
# if(exists($exp_h->{$w}))
# {
# print "$w\n";
# next;}
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each($hash->{$w}))
{
if(exists($hash_st->{$k}))
{next;}
$tmp_hash->{$k}+=$v;;
#$tmp_hash->{$k}=$v;
#$tmp_hash->{$k}=$hash_sc->{$w}->{'h'};
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
#正常词 交 转换0.00--0.00 相互
#if(exists($sytmp->{$k}))
#{
# print "归一 $k ",$sytmp->{$k},"\n";
# $k=$sytmp->{$k};
#}
# if($value>20)
# $heap->{client}->put("$k $v\n");
}
}
}
return (\%{$tmp_hash},$nline);
}
#print "output = $res\n";
# $heap->{client}->stop();
# $session->stop();
# Start the server.
sub comp_hash
{
my $h1=@_[0];
my $h2=@_[1];
my $akv=0;
my $all;
my %all;
my $vall=0.001;
my $vall2=0.001;
my $k,$v;
while(($k,$v)=each(%{$h1}))
{
$all->{$k}++;
$vall+=$v;
print FD_log " hash 1$k $v\n";
}
my $k,$v;
while(($k,$v)=each(%{$h2}))
{
$all->{$k}++;
print FD_log " hash 2$k $v\n";
$vall2+=$v;
}
my $k,$v;
while(($k,$v)=each(%{$all}))
{
if($v >= 2)
{
# my $sum;
#if($h1->{$k}>$h2->{$k})
#{$sum=$h2->{$k};}
#else
#{$sum=$h1->{$k};}
#$kv=$sum/($vall+$vall2);
$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
#$kv2=(($sum/$vall) + ($sum/$vall2))/2;
$kv2=((($h1->{$k})/$vall) + ($h2->{$k}/$vall2))/2;
$kv3=($kv+$kv2)/2;
#if($h1->{$k}<$h2->{$k})
#{
#}
$akv+=$kv3;
print FD_log "double $k $v kv sum= $h1->{$k}+$h2->{$k} kv= $kv val1= $vall val2= $vall2 kv2=$kv2 kv3 =$kv3\n";
}
}
# print "---------------\n";
$akv;
}
sub ndate
{
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time());
$year=~s/1(.*)/20$1/;
$mon+=1;
if($mon=~/^\d$/)
{
$mon="0"."$mon";
}
if($mday=~/^\d$/)
{
$mday="0"."$mday";
}
if($hour=~/^\d$/)
{
$hour="0"."$hour";
}
if($min=~/^\d$/)
{
$min="0"."$min";
}
if($sec=~/^\d$/)
{
$sec="0"."$sec";
}
my $res=$year."-".$mon."-".$mday." ".$hour.":"."$min".":"."$sec";
#print $res,"\n";
$res;
}
sub txt2arr_one
{
my $w=$_[0];
my $exp_h=$_[1];
$tmp_hash=();
$rand=rand();
@res=();
$part=0;
#print @res;
my @w_arr=split(",",$w);
foreach my $w_tmp (@w_arr)
{
if($w_tmp=~/(.*?) (.*)/)
{
$w=$1;
$vh=$2;
#print "$w -- $vh\n";
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each($hash->{$w}))
{
if($v==0)
{
$v=1;
}
$tmp_hash->{$k}=$vh;
}
}
$tmp_hash->{$w}=$w;
}
else
{
print "input error\n";
}
}
# while(($k,$v)=each(%{$tmp_hash}))
#
# {
#print "meaning $k $v\n";
# }
# }
return (%{$tmp_hash});
}
sub txt2arr3
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
my @list=split(" ",$input_t);
print FD_log "$input_t\n";
@res=();
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
my $l=length($tmp_res);
if($l<=3)
{next;}
#原始词保留
#主题放大
$w=$tmp_res;
$tmp_hash->{$w}+=2;
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each($hash->{$w}))
{
$tmp_hash->{$k}+=$v;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
# if($hash_sc->{$w}->{'h'}=~/n/)
}
}
else
{
$tmp_hash->{$w}+=0.1;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
}
}
#是否采用scws放大文章中心词
# while(($k,$v)=each(%{$tmp_hash}))
# {
# if(($k eq $n1)||($k eq $n2)||($k eq $v1)||($k eq $v2))
# {$tmp_hash->{$k}=$v*3;
# print FD_log "放大主题-$k-",$tmp_hash->{$k}," \n";
# }
# }
return (\%{$tmp_hash},$nline);
}
sub search_article2
{
my $file=$_[0];
chomp($file);
#print "file=$file\n";
$expword=$ARGV[1];
$exp_hash=();
@exp_w=split(",",$expword);
foreach $exp (@exp_w)
{
$exp_hash->{$exp}=1;
}
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %thash;
my $j;
#my $rand2=rand();
#open(FD_hash,">$file.txt");
#print "openfile $file\n";
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
while(<FD_arr>)
{
my $in=$_;
chomp($in);
print FD_aro "$j ",$in,"\n";
$j++;
}
close FD_arr;
close FD_aro;
system("/home/wuyabo/xs/bin/scws -I -E -N -d /home/wuyabo/xs/etc/dict.utf8.xdb:/home/wuyabo/stanford/dict_user.txt -c utf-8 /tmp/$rand.sctxt -o /tmp/$rand.sctxt2");
unlink("/tmp/$rand.sctxt");
open(FD_scws,"/tmp/$rand.sctxt2");
my $hashleft;
my $alln=1;
while(<FD_scws>)
{
my $in=$_;
print FD_log "line ttt $kk $in \n";
chomp($in);
my $n_line;
my ($my_hash2,$n_line)=txt2arr($in,$exp_hash);
$hashleft[$n_line]=clone($my_hash2);
$alln++;
}
for( 1 .. $alln-1)
{
my $nn=$_;
for ($nn .. $alln-1)
{
$nn2=$_;
if($nn==$nn2)
{next;}
$n=comp_hash($hashleft[$nn],$hashleft[$nn2]);
$n=$n * 100;
print "$n $nn $nn2 \n";
print FD_log "$n $nn $nn2 \n";
}
}
unlink("/tmp/$rand.sctxt2");
close(FD_arr);
@m_res;
}