zhangbo-NLP-plugin_eng/lib/zhangbo/NLP/eng_org813.pl
#!/usr/bin/perl
use lib "./";
use plugin_eng;
use POE qw(Component::Server::TCP);
#use JSON::XS;
use Clone qw(clone);
#use MongoDB;
#use Smart::Comments;
#use Conn_mongo_jc;
use Data::Dumper;
#use IO::File;
#use Add_info;
our @rewrite_g;
our @rewrite_l;
our @rewrite_n;
our %rewrite_all;
our $rewrite_all;
#system("./test_files2.py 1>/dev/null 2>/dev/null &");
#system("./post_server.pl &");
#open(FDst,"stop.txt");
#our $hash_st;
#while(<FDst>)
#{
# my $line;
# $line=$_;
# chomp($line);
# $hash_st->{$line}=1;
#}
#close FDst;
# http://poe.perl.org/?POE_Cookbook/TCP_Servers
# Include POE and POE::Component::Server::TCP.
our $hash=();
$hash_mn=();
my $hash_sc=();
#open(FD_sc, "dict.utf8.txt");
open(FD_log, ">/tmp/engine.txt");
#while(<FD_sc>)
#{
#招收学员 13.87 8.50 n
# my $line=$_;
# if($line=~/(.*?)\t(.*?)\t(.*?)\t(.*)/)
# {
# $tt1=$1;
# $tt2=$2;
# $tt3=$3;
# $tt4=$4;
# $hash_sc->{$tt1}->{"i"}=$tt2;
# $hash_sc->{$tt1}->{"h"}=$tt3;
# $hash_sc->{$tt1}->{"a"}=$tt4;
#
# }
#}
#system("pwd");
#system("ls ../");
#print "@INC\n";
#print "$INC[1]\n";
#if(! -e "./word.txt")
#{print "dic file found ./\n";
#};
#if(! -e "$INC[1]/word.txt")
#{print "dic file found /\n";
#};
#if(! -e "../word.txt")
#{print "dic file found ../\n";
#};
#if(! -e "/home/lzj/shell2/word.txt")
#{print "dic file found /home/lzj/shell2/\n";
#};
open(FD, "sy.txt");
my $sytmp;
while(<FD>)
{
my $line=$_;
if($line=~/\#(.*)/)
{
$tsy=$1;
if(!exists($sytmp->{$tsy}))
{
$sytmp->{$tsy}=$tsy;
$eat=1;
# print "id=$tsy\n";
}
else
{
$eat=0;
}
}
elsif($eat==1)
{
chomp($line);
if(length($tsy)<=3)
{next;}
if(length($line)<=3)
{next;}
$hash->{$line}->{$tsy}=0.2;
# $hash->{$tsy}->{$line}=0.2;
# print "-$tsy-$line\n";
}
}
close FD;
open(FD, "word.txt");
my $tmp=();
while(<FD>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
chomp($line);
if($line=~/^W_C=(.*)/)
{
$tmp=$1;
#print $tmp,"\n";
$r=1;
}
elsif($line=~/^DEF=(.*)/)
{
$r=0;
$tmp_m=$1;
# print $tmp_m,"\n";
if(length($tmp)<=3)
{next;}
if(exists($hash_st->{$tmp}))
{next;}
my @arry=split('\|',$tmp_m);
foreach $tmps (@arry)
{
# print "-----------------$tmps ------------\n";
if($tmps=~/(.*?)\:/)
{
my $m=$1;
if(exists($hash_st->{$m}))
{next;}
# if($m eq "专")
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
# print "$tmp - $m-----\n";
if(exists($hash_sc->{$m}))
{
if($r==0)
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
}
else{ $hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*0.2;}
#$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.1;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
$r++;
}
elsif($tmps=~/(.*?)\}/)
{
my $m=$1;
# if($m eq "专")
#if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件"))
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
if(exists($hash_st->{$m}))
{next;}
if($r==0)
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
}
else
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*(0.5);
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=0.5;
}
$r++;
}
}
}
}
open(FDb, "babel_all_path_fin20.txt");
my $tmp=();
while(<FDb>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
$s=$line;
# $m=$2;
my $name;
@a=split("\t",$s);
my $dob=0;
#@b=split(" ",$m);
$f=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$name=$i;
if(length($name)<3)
{last;}
if(exists($hash->{$name}))
{
#print "$name\n";
$dob=1;
# last;
}
if(exists($hash_st->{$name}))
{last;}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
if(exists($hash->{$name}->{$av}))
{next;}
if($dob==1)
{
$hash->{$name}->{$av}=$bv*0.3;
}
else
{
$hash->{$name}->{$av}=$bv;
}
}
}
}
#foreach $i (@b)
#{
# $hash->{$name}->{$i}+=0.01;
#}
}
close(FDb);
sub reloaddic
{
open(FDst,"stop.txt");
our $hash_st;
while(<FDst>)
{
my $line;
$line=$_;
chomp($line);
$hash_st->{$line}=1;
}
close FDst;
load_dic("babel_add.txt",60,0.5);
load_dic("名词.txt",20,0.5);
load_dic("动词.txt",10,0.5);
load_dic("zw.txt",10,0.2);
}
sub load_dic
{
open(FDb, "$_[0]");
my $lsize=$_[1];
my $zoomin=0.5;
if($_[2]>0)
{$zoomin=$_[2];}
my $tmp=();
while(<FDb>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
$s=$line;
# $m=$2;
my $name;
@a=split("\t",$s);
my $dob=0;
#@b=split(" ",$m);
$f=0;
my $ccn=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$name=$i;
#if(exists($hash->{$name}))
#{
#print "$name\n";
# $dob=1;
# last;
#}
# if(exists($hash_st->{$name}))
# {last;}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
#覆盖
if(exists($hash_st->{$av}))
{next;}
if($ccn>$lsize)
{last;}
$ccn++;
$hash->{$name}->{$av}=$bv*$zoomin;
# print "$name $av $bv\n";
}
}
}
#foreach $i (@b)
#{
# $hash->{$name}->{$i}+=0.01;
#}
}
close (FDb);
print "load $_[0] ok\n";
}
close FD_log;
load_dic("babel_add.txt",60,0.5);
load_dic("名词.txt",20,0.5);
load_dic("动词.txt",10,0.5);
load_dic("zw.txt",10,0.2);
#reloaddic();
load_rewrite();
# print Data::Dumper->Dump([%{$rewrite_g[1]}]);
#print Data::Dumper->Dump([%{$hash}]);
#print "-----\n";
# print Data::Dumper->Dump([%{$rewrite_all}]);
#print "-----\n";
print "server start\n";
my $n=0;
#our $f=0;
POE::Component::Server::TCP->new(
Alias => "echo_server",
Port => 11221,
ClientInput => sub {
my ($session, $heap, $input) = @_[SESSION, HEAP, ARG0];
#print "Session ", $session->ID(), " got input: ".$input."\n";
#print "Session ", $session->ID(), " got input: ".length($input)."\n";
my $d;
my $d_seq;
open(FD_log,">/tmp/engine.log");
print FD_log "$input\n";
#system("cp $input /tmp/");
print "input file $input",length($input),"\n";
my @res;
my $noshudown=0;
if ($input=~/^reload_dic/)
{
#@res=search_article2($input);
reloaddic();
load_rewrite();
#my $psid=`ps aux|grep test_files2.py|grep -v grep|awk '{print \$2}'`;
#if($psid >0)
#{
#system("kill -9 $psid");
#print ("kill $psid\n");
#}
#system("./test_files2.py 1>/dev/null 2>/dev/null &");
#my $psid=`ps aux|grep test_files2.py|grep -v grep|awk '{print \$2}'`;
#print $psid,"\n";
#print FD_log "./test_files2.py 1>/dev/null 2>/dev/null &\n";
}
elsif ($input=~/^reload_plugin (.*)/)
{
my $kkk=$1;
delete $INC{"$kkk.pm"};
eval("require $kkk;");
print "require $kkk\n";
}
elsif($input=~/(.*) pinyin/)
{
#print "num ----------------- start\n";
my $kkk=$1;
@res=plugin_eng::pinyin($kkk);
}
elsif($input=~/(.*) num_n (\d+) (\d+)/)
{
#print "num ----------------- start\n";
my $kkk=$1;
@res=plugin_eng::num_n($kkk,$2,$3);
}
elsif($input=~/dist (.*?) (.*)/)
{
#直接给出 2个句子/词的 距离
@res=dist($1,$2);
$noshudown="yes";
}
elsif($input=~/.* deep/)
{
#深度分析间接语义联系
@res=deep_article($input);
}
elsif($input=~/(.*) path/)
{
my $kkk=$1;
my @res1=mean_path($input);
my @res2=deep_article_path($kkk);
@res=(@res1,@res2);
}
else
{
#@res=deep_article($input);
@res=search_article($input);
}
print FD_log @res;
close(FD_log);
#print scalar(@res)," num\n";
$heap->{client}->put(@res);
#$session->yield("shutdown");
if($noshudown eq "yes")
{
}
else
{
$_[KERNEL]->yield("shutdown");
return;
}
},
ClientDisconnected => sub {
#print "Client disconnected\n"; # log it
}
#, ClientFlushed => sub {
# my $data_source = $_[HEAP]{file_handle};
# my $read_count = sysread($data_source, my $buffer = "", 65536);
# if ($read_count) {
# $_[HEAP]{client}->put($buffer);
# }
# else {
# print FD_log "------------------error\n";
# $_[KERNEL]->yield("shutdown");
# }
#}
);
POE::Kernel->run;
#$file=$ARGV[0];
sub mean_path
{
my $file=$_[0];
chomp($file);
$file=~s/ path//g;
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash;
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %my_hash2_seq;
my %thash;
my $j;
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
my @inputa;
my @inputb;
while(<FD_arr>)
{
my $in=$_;
if($first==0)
{
if($in=~/^-/)
{
$first++;
print FD_aro "$in";
}
else{
print FD_aro "0 $in";
push (@inputa,$in);
}
}
else
{
print FD_aro "$j $in";
push (@inputb,$in);
$j++;
}
}
close FD_arr;
close FD_aro;
#system("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 ");
system("cp -a /tmp/$rand.sctxt /tmp/$rand.sctxt2 ");
#print("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 2>/dev/null 1>/dev/null");
unlink("/tmp/$rand.sctxt");
open(FD_scws,"</tmp/$rand.sctxt2");
print FD_log "open /tmp/$rand.sctxt2 \n";
my $disthash;
my @disthash;
my $dist_seq_hash;
my @dist_seq_hash;
my $in=<FD_scws>;
#foreach my $in (@output)
chomp($in);
$in=~s/0 (.*)/$1/g;
my $first=$in;
print FD_log "first in= $first\n";
my $innext=<FD_scws>;
print FD_log "innext = $innext\n";
my $to=<FD_scws>;
#foreach my $in (@output)
chomp($to);
$to=~s/1 (.*)/$1/g;
#$to=~s/.* (.*)/$1/g;
print FD_log "to=$to\n";
my $path_road;
my @m_res;
print FD_log "deep in= $in\n";
$in1="0 ".$in;
($d,$c,$d_seq,$og)=txt2arr_input($in1,$exp_hash);
my $deep;
$total1=0;
$total2=0;
$total3=0;
$total4=0;
$total5=0;
$total6=0;
while(my ($k,$v)=each($d))
{
$total1++;
print FD_log "deep k=$k| to=$to|\n";
if($k eq $to)
{
print FD_log "patth "."$in-$to-"."直接相关 $v\n";
push (@m_res,"$in-$to-"."直接相关 $v");
}
print FD_log "debug -$k-$to-\n";
$k1="0 ".$k;
#if(exists($path_road->{$k."-".$deek}))
#{next;}
my ($dee,$deec,$deed_seq,$deeog)=txt2arr_input($k1,$exp_hash);
while(($deek,$deev)=each($dee))
{
$total2++;
print FD_log "deep deek2= $deek\n";
# if(exists($path_road->{$k."-".$deek}))
# {next;}
if($deek eq $to)
{
print FD_log "patth $in-$to-"."1层 ->$k ".$v." "."$deev\n";
push (@m_res,"$in-$to-"."1层->$k -> $deek :".$v." ".$deev);
}
$deek1="0 ".$deek;
my ($deek2,$deec2,$deed_seq2,$deeog2)=txt2arr_input($deek1,$exp_hash);
if(exists($hash->{$deek}))
{
while(($deek3,$deev3)=each($deek2))
{
$total3++;
print FD_log "deep deek3= $deek3\n";
if(exists($path_road->{$deek."-".$deek3}))
{next;}
if($deek3 eq $to)
{
#print FD_log "patth $in-$to-"."2层通过$k和$deek:".$v." ".$deev." "."$deev3\n";
push (@m_res,"$in-$to-"."2层 ->$k -> $deek -> $deek3 :".$v." ".$deev." ".$deev3);
}
$path_road->{$deek."-".$deek3}=1;
}
}
$path_road->{$k."-".$deek}=1;
}
$path_road->{$in."-".$k}=1;
}
$to1="0 ".$to;
my ($d,$c,$d_seq,$og)=txt2arr_input($to1,$exp_hash);
my $deep;
# if(exists($hash->{$to}))
# {
while(my ($k,$v)=each($d))
{
$total4++;
print FD_log "re deep k=$k in=$in\n";
if($k eq $in)
{
print FD_log "re patth "."$to-$in-"."直接相关 $v\n";
push (@m_res,"$to-$in-"."直接相关 $v");
}
print FD_log "re debug -$k-$in-\n";
$k1="0 ".$k;
my ($dee,$deec,$deed_seq,$deeog)=txt2arr_input($k1,$exp_hash);
# if(exists($hash->{$k}))
# {
while(($deek,$deev)=each($dee))
{
$total5++;
print FD_log "re deep2 deek= $deek\n";
# if(exists($path_road->{$k."-".$deek}))
# {next;}
if($deek eq $in)
{
print FD_log "re patth $to-$in-"."1层->$k ".$v." "."$deev\n";
push (@m_res,"$to-$in-"."1层-> $k -> $deek :".$v." ".$deev);
#
}
$deek1="0 ".$deek;
my ($deek2,$deec2,$deed_seq2,$deeog2)=txt2arr_input($deek1,$exp_hash);
while(($deek3,$deev3)=each($deek2))
{
$total6++;
print FD_log "re deep3 deek3= $deek3\n";
if(exists($path_road->{$deek."-".$deek3}))
{next;}
if($deek3 eq $in)
{
print FD_log "re patth $to-$in-"."2层通过$k和$deek:".$v." ".$deev." "."$deev3\n";
push (@m_res,"$to-$in-"."2层-> $k -> $deek -> $deek3 :".$v." ".$deev." ".$deev3);
}
$path_road->{$deek."-".$deek3}=1;
}
$path_road->{$k."-".$deek}=1;
}
#}
$path_road->{$to."-".$k}=1;
}
push (@m_res,"循环次数 .$total1 $total2 $total3 $total4 $total5 $total6");
@m_res;
}
sub deep_article
{
my $file=$_[0];
chomp($file);
$file=~s/ deep//g;
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash;
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %my_hash2_seq;
my %thash;
my $j;
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
my @inputa;
my @inputb;
while(<FD_arr>)
{
my $in=$_;
if($first==0)
{
if($in=~/^-/)
{
$first++;
print FD_aro "$in";
}
else{
print FD_aro "0 $in";
push (@inputa,$in);
}
}
else
{
print FD_aro "$j $in";
push (@inputb,$in);
$j++;
}
}
close FD_arr;
close FD_aro;
system("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 ");
print("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 2>/dev/null 1>/dev/null");
unlink("/tmp/$rand.sctxt");
open(FD_scws,"</tmp/$rand.sctxt2");
print FD_log "open /tmp/$rand.sctxt2 \n";
my $disthash;
my @disthash;
my $dist_seq_hash;
my @dist_seq_hash;
while(<FD_scws>)
#foreach my $in (@output)
{
my $in=$_;
print FD_log "start \n";
print FD_log "line ttt $kk $in \n";
chomp($in);
#$input_ar[$f]=$in;
if($f==0)
{
if(($in=~/^- -/)||($in=~/^ - -/))
{
$f++;
next;
}
#if(length($in)>12)
#{
$in=rewrite_gx($in);
print "in == $in\n";
($d,$c,$d_seq,$og)=txt2arr_input($in,$exp_hash);
my $deep;
while(($k,$v)=each($d))
{
$k="0 ".$k;
my ($dee,$deec,$deed_seq,$deeog)=txt2arr_input($k,$exp_hash);
while(($deek,$deev)=each($dee))
{
$deep->{$deek}+=$deev;print FD_log "deep $deek $deev\n";}
}
$disthash[$kk]=clone($deep);
# $disthashog[$kk]=clone($og);
# $dist_seq_hash[$kk]=clone($d_seq);
$kk++;
#}
next;
#print Data::Dumper->Dump([%{$d}]);
# %my_hash2=txt2arr($input2);
# $n=comp_hash(\%my_hash,\%my_hash2);
}
else
{
my $n_line;
$in=rewrite_gx($in);
print "in2 == $in\n";
my ($my_hash2,$n_line,$my_hash2_seq,$og2)=txt2arr3($in,$exp_hash);
for (0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
$n=comp_hash($disthash[$nnn],$my_hash2,$dist_seq_hash[$nnn],$my_hash2_seq,$disthashog[$nnn],$og2);
my $shnn=$nnn+1;
print FD_log "totle $in $shnn-$n_line=$n\n";
print FD_log "$inputa[$nnn]\n$inputb[$n_line-1]\n";
$thash[$nnn]->{$n_line}=$n;
}
}
$f++;
}
#unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
close(FD_arr);
my @m_res;
for(0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
my @key =sort {$thash[$nnn]->{$b} <=> $thash[$nnn]->{$a}} keys %{$thash[$nnn]};
#print @key;
my $o_n;
foreach $i (@key)
{
# if( $thash[$nnn]->{$i}>0)
# {
my $shnnn=$nnn+1;
push (@m_res,"$shnnn-$i-".$thash[$nnn]->{$i});
#push (@m_res,"$nnn-$i-".$thash[$nnn]->{$i}.$input_ar[$i]);
# $o_n++;
# }
}
}
print FD_log "return ",scalar(@m_res),"\n";
@m_res;
}
sub search_article
{
$file=$_[0];
chomp($file);
#print "file=$file\n";
$expword=$ARGV[1];
$exp_hash=();
@exp_w=split(",",$expword);
foreach $exp (@exp_w)
{
$exp_hash->{$exp}=1;
}
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash;
my @input_ar=`cat $file`;
my %my_hash;
my %my_hash2;
my %my_hash2_seq;
my %thash;
my $j;
#my $rand2=rand();
#open(FD_hash,">$file.txt");
#print "openfile $file\n";
open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
my @inputa;
my @inputb;
while(<FD_arr>)
{
#print FD_log $j,"----------\n";
#print $j,"----------\n";
my $in=$_;
if($first==0)
{
if($in=~/^-/)
{
$first++;
print FD_aro "$in";
}
else{
print FD_aro "0 $in";
push (@inputa,$in);
}
}
else
{
#if(length($in)>500)
#{ $in=substr $in,0,500;}
# $in=~s/\@.*?\:/ /g;
# $in=~s/\@.*?\s/ /g;
# $in=~s/机场|航班|深圳/ /g;
my $jid;
if($in=~/^id=(.*?) /)
{
$jid=$1;
}
else{$jid=$j;}
print FD_aro "$jid $in";
push (@inputb,$in);
$j++;
#print FD_log $j,"-p----",length($in),"\n";
#print $j,"-p----",length($in),"\n";
}
}
close FD_arr;
close FD_aro;
system("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 ");
print("./client_qc.pl /tmp/$rand.sctxt /tmp/$rand.sctxt2 2>/dev/null 1>/dev/null");
#system("/home/lzj/shell/jieba-0.31/test/test_file.py /tmp/$rand.sctxt /tmp/$rand.sctxt2 2>/dev/null 1>/dev/null");
#system("/home/wuyabo/xs/bin/scws -I -E -N -d /home/wuyabo/xs/etc/dict.utf8.xdb:/home/wuyabo/stanford/dict_user.txt -c utf-8 /tmp/$rand.sctxt -o /tmp/$rand.sctxt2");
#system("cp /tmp/$rand.sctxt2 66.txt");
#system("cp /tmp/$rand.sctxt 55.txt");
#system("cp $file 44.txt");
#print ("cp ok\n");
unlink("/tmp/$rand.sctxt");
open(FD_scws,"</tmp/$rand.sctxt2");
print FD_log "open /tmp/$rand.sctxt2 \n";
my $disthash;
my @disthash;
my $dist_seq_hash;
my @dist_seq_hash;
while(<FD_scws>)
#foreach my $in (@output)
{
my $in=$_;
print FD_log "start \n";
print FD_log "line ttt $kk $in \n";
chomp($in);
#$input_ar[$f]=$in;
if($f==0)
{
if(($in=~/^- -/)||($in=~/^ - -/))
{
$f++;
next;
}
#if(length($in)>12)
#{
$in=rewrite_gx($in);
print FD_log "rewrite == $in\n";
($d,$c,$d_seq,$og)=txt2arr_input($in,$exp_hash);
$disthash[$kk]=clone($d);
$disthashog[$kk]=clone($og);
$dist_seq_hash[$kk]=clone($d_seq);
$kk++;
#}
next;
#print Data::Dumper->Dump([%{$d}]);
# %my_hash2=txt2arr($input2);
# $n=comp_hash(\%my_hash,\%my_hash2);
}
else
{
my $n_line;
$in=rewrite_gx($in);
print "in2 == $in\n";
my ($my_hash2,$n_line,$my_hash2_seq,$og2)=txt2arr3($in,$exp_hash);
# while (($k,$v)=each(%my_hash2))
# {
# print FD_hash "$k,$v\t";
# }
# print FD_hash "\n";
for (0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
$n=comp_hash($disthash[$nnn],$my_hash2,$dist_seq_hash[$nnn],$my_hash2_seq,$disthashog[$nnn],$og2);
my $shnn=$nnn+1;
print FD_log "totle $in $shnn-$n_line=$n\n";
print FD_log "$inputa[$nnn]\n$inputb[$n_line-1]\n";
$thash[$nnn]->{$n_line}=$n;
}
}
$f++;
}
#unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
close(FD_arr);
my @m_res;
for(0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
my @key =sort {$thash[$nnn]->{$b} <=> $thash[$nnn]->{$a}} keys %{$thash[$nnn]};
#print @key;
my $o_n;
foreach $i (@key)
{
# if( $thash[$nnn]->{$i}>0)
# {
my $shnnn=$nnn+1;
push (@m_res,"$shnnn-$i-".$thash[$nnn]->{$i});
#push (@m_res,"$nnn-$i-".$thash[$nnn]->{$i}.$input_ar[$i]);
# $o_n++;
# }
}
}
print FD_log "return ",scalar(@m_res),"\n";
@m_res;
}
# }
# );
#$poe_kernel->run();
sub txt2arr
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
# chomp($input_t);
my @list=split(" ",$input_t);
#print "txt2arr input =$input_t\n";
#$rand=rand();
#open(FD_r,">/tmp/$rand");
#print FD_log "$input_t\n";
#close(FD_r);
#$dir="/home/wyb/stanford-parser-2012-07-09";
@res=();
#@res=`$dir/parser-10.pl /tmp/$rand 0 50`;
#$part=0;
#unlink("/tmp/$rand");
#print @res;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
if(exists($hash_st->{$tmp_res}))
{next;}
my $l=length($tmp_res);
# if($l<=3)
# {
#if($hash_sc->{$tmp_res}->{'a'} !~/n|v|a/)
#{
#print FD_log "seek $tmp_res\n";
#next;}
if($l<3)
{next;}
#主题放大
$tmp_hash->{$tmp_res}++;
# }
# print $tmp_res,"---",length($tmp_res),"\n";
#if (($part==1)&&($tmp_res=~/(.*?),(.*)/))
#{
#print "切词1=$1,2=$2\n";
my $w=$tmp_res;
# my $value=$2;
# if(exists($exp_h->{$w}))
# {
# print "$w\n";
# next;}
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
if(exists($hash_st->{$k}))
{next;}
$tmp_hash->{$k}+=$v;;
#$tmp_hash->{$k}=$v;
#$tmp_hash->{$k}=$hash_sc->{$w}->{'h'};
# print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
#正常词 交 转换0.00--0.00 相互
#if(exists($sytmp->{$k}))
#{
# print "归一 $k ",$sytmp->{$k},"\n";
# $k=$sytmp->{$k};
#}
# if($value>20)
# $heap->{client}->put("$k $v\n");
}
}
}
return (\%{$tmp_hash},$nline);
}
#print "output = $res\n";
# $heap->{client}->stop();
# $session->stop();
# Start the server.
sub comp_hash
{
my $h1=@_[0];
my $h2=@_[1];
my $h3=@_[2];
my $h4=@_[3];
my $org1=@_[4];
my $org2=@_[5];
#while(($k,$v)=each(%{$h3}))
# {
#print FD_log "h3 show $k, $v\n";
# }
#while(($k,$v)=each(%{$h4}))
# {
#print FD_log "h4 show $k, $v\n";
# }
my $akv=0;
my $all;
my $an_all;
my %all;
my $vall=0.001;
my $vall2=0.001;
my $k,$v;
while(($k,$v)=each(%{$h1}))
{
$all->{$k}++;
if($v>0)
{
$an_all++;
$vall+=$v;
}
print FD_log " hash 1 k $k v $v\n";
}
my $k,$v;
while(($k,$v)=each(%{$h2}))
{
$all->{$k}++;
if($v>0)
{
$an_all++;
#$vall2++;
$vall2+=$v;
}
print FD_log " hash 2 k $k v $v\n";
}
my $k,$v;
my $samelist;
my $std_n;
while(($k,$v)=each(%{$all}))
{
if($v >= 2)
{
# my $sum;
#if($h1->{$k}>$h2->{$k})
#{$sum=$h2->{$k};}
#else
#{$sum=$h1->{$k};}
#$kv=$sum/($vall+$vall2);
if($h1->{$k}<0)
{next;}
$an_all=$an_all-2;
# 平均算法 适用于多词找文 对词典中噪音有包容性 #$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
#$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
$kv2=((($h1->{$k})/$vall) + ($h2->{$k}/$vall2))/2;
$kv3=($kv+$kv2)/2;
# log2算法 适用于文找文 $kv=($h1->{$k}+$h2->{$k})/($vall+$vall2+log2($an_all));
#$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2+log2($an_all)/2);
#$kv3=$kv;
#$kv2=((($h1->{$k})/$vall) + ($h2->{$k}/$vall2))/2;
#if($h1->{$k}<$h2->{$k})
#{
#}
$akv+=$kv3;
print FD_log "double $k $v kv sum= $h1->{$k}+$h2->{$k} ($org1->{$k}, $org2->{$k}) kv= $kv val1= $vall val2= $vall2 kv2=$kv2 kv3 =$kv3 std_n=$std_n\n";
print FD_log "相似语义 $k 来自前句 from ";
#while(($kc,$vc)=each($org1->{$k}))
#{
# print "$kc,";
#}
#print " 来自后句 ";
if (exists($org2->{$k}))
{
while(($kc,$vc)=each($org2->{$k}))
{
print "$kc,";
}
print "\n";
}
$samelist->{$k}=$h3->{$k};
$std_n++;
}
}
#序列相似评估
#---------------------------------------
if(($std_n>0)&&($std_n !=1))
{
# make samelist ->std test_seq->obj_seq;
my @key =sort {$samelist->{$b} <=> $samelist->{$a}} keys %{$samelist};
#print @key;
my $std_seq;
my $t_n=1;
my $test_seq;
my $last_tmp;
my $my_fi=0;
my $std_sq;
foreach $i (@key)
{
if(exists($std_sq->{$samelist->{$i}}))
{
$std_seq->{$i}=($std_n-$t_n+1);
$test_seq->{$i}=$h4->{$i};
# print FD_log "def test_seq $i =$h4->{$i}\n";
# print FD_log "std_seq last $i seq =$std_seq->{$i} h3 $h3->{$i} std_n=$std_n\n";
}
else
{
$t_n++;
$std_sq->{$samelist->{$i}}=1;
$std_seq->{$i}=($std_n-$t_n+1);
# print FD_log "std_seq $i seq=$std_seq->{$i}\n";
# print FD_log "def test_seq $i =$h4->{$i}\n";
$test_seq->{$i}=$h4->{$i};
}
}
my @key =sort {$test_seq->{$b} <=> $test_seq->{$a}} keys %{$test_seq};
my $t_n=1;
my $last_tmp;
my $obj_tmp;
my $test_sq;
foreach $i (@key)
{
print FD_log "$i \"$last_tmp\" == \"$test_seq->{$i}\" ??\n";
# if($last_tmp eq $test_seq->{$i})
if(exists($test_sq->{$test_seq->{$i}}))
{
$obj_seq->{$i}=($std_n-$t_n+1);
# print FD_log "test_seq last $i seq= $obj_seq->{$i} std_n=$std_n\n";
}
else
{
$t_n++;
$test_sq->{$test_seq->{$i}}=1;
$obj_seq->{$i}=($std_n-$t_n+1);
}
# test_seq 受体拮抗剂 last_tmp = 26 25
# print FD_log "test_seq $i seq= $obj_seq->{$i}\n";
}
my $diff_sum=0;
while(($k,$v)=each(%{$std_seq}))
{
$diff_sum=$diff_sum+abs($v-$obj_seq->{$k});
print FD_log "diff_sum $diff_sum $k $v $obj_seq->{$k}\n";
}
my $sam_rate=1-($diff_sum/(($std_n-1)*($std_n-1)));
my $diff_n=$akv*(1-$sam_rate)*0.15;
$akv=$akv-$diff_n;
print FD_log "seq diff sam_rate $sam_rate akv $akv diff_n $diff_n\n";
}
#-----------------------------------------------
# 异议评估 说多了远的，深意不在这里
# my $maxd=log2($an_all+1);
# if($maxd>100)
# {$akv=$akv*0.618;}
# elsif($maxd==0)
# {
# }
# elsif($maxd>1)
# {
# $akv=$akv*(0.8+0.2/($maxd+1));
# }
# print "---------------\n";
$akv;
}
sub ndate
{
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time());
$year=~s/1(.*)/20$1/;
$mon+=1;
if($mon=~/^\d$/)
{
$mon="0"."$mon";
}
if($mday=~/^\d$/)
{
$mday="0"."$mday";
}
if($hour=~/^\d$/)
{
$hour="0"."$hour";
}
if($min=~/^\d$/)
{
$min="0"."$min";
}
if($sec=~/^\d$/)
{
$sec="0"."$sec";
}
my $res=$year."-".$mon."-".$mday." ".$hour.":"."$min".":"."$sec";
#print $res,"\n";
$res;
}
sub txt2arr_one
{
my $w=$_[0];
my $exp_h=$_[1];
$tmp_hash=();
$rand=rand();
@res=();
$part=0;
#print @res;
my @w_arr=split(",",$w);
foreach my $w_tmp (@w_arr)
{
if($w_tmp=~/(.*?) (.*)/)
{
$w=$1;
$vh=$2;
#print "$w -- $vh\n";
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
if($v==0)
{
$v=1;
}
$tmp_hash->{$k}=$vh;
}
}
$tmp_hash->{$w}=$w;
}
else
{
print "input error\n";
}
}
# while(($k,$v)=each(%{$tmp_hash}))
#
# {
#print "meaning $k $v\n";
# }
# }
return (%{$tmp_hash});
}
sub txt2arr_input
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
my $tmp_sequ;
my $sequ=0;
my @list=split(" ",$input_t);
print FD_log "$input_t\n";
@res=();
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
my $og;
my $linst_len;
$list_len=scalar(@list);
for (0 .. ($list_len-1))
#foreach $tmp_res (@list)
{
$nown=$_;
$tmp_res=$list[$_];
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
my $l=length($tmp_res);
if(($l<3)&&($l!~/^[a-z|A-Z]/))
{next;}
if(exists($hash_st->{$tmp_res}))
{next;}
if($tmp_res=~/^\d/)
{next;}
if($tmp_res=~/^-/)
{next;}
#原始词保留
#主题放大
my $zs=1;
#-----------------------------------------------------------------------
$w=$tmp_res;
if(($list[$nown+1]=~/^\d/)&&(length($input_t)<10))
{
$tmp_hash->{$w}+=2*$list[$nown+1];
$zs=$list[$nown+1];
}
elsif(($list[$nown+1]=~/^-/))
{
$tmp_hash->{$w}-=2*$list[$nown+2];
$zs=-$list[$nown+2];
}
else
{
#控制重复词语数量
if($tmp_hash->{$w}>($list_len/10))
{next;}
else
{$tmp_hash->{$w}+=2;
$og->{$w}->{$w}=1;
}
$tmp_sequ->{$w}=$sequ;
$sequ++;
}
#----------------------------------------------------------
##序列
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
$tmp_hash->{$k}+=$v*$zs;
$og->{$k}->{$w}=1;
$tmp_sequ->{$k}=$sequ;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k},"k=$k sequ=$sequ\n";
}
}
else
{
$tmp_hash->{$w}+=0.1;
$og->{$w}->{$w}=1;
$tmp_sequ->{$k}=$sequ;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k},"k=$k sequ=$sequ\n";
}
}
return (\%{$tmp_hash},$nline,\%{$tmp_sequ},\%{$og});
}
sub txt2arr3
{
#需要改进为 先判断文本域 再根据域 重写词向量值 域向量集 可以用clone覆盖已有向量
#例如黑话库 的词依存 规则
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
my $tmp_hash_seq=();
my @list=split(" ",$input_t);
print FD_log "$input_t\n";
@res=();
my $og;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $seq=0;
my $tmp_first=0;
my $list_len;
$list_len=scalar(@list);
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
my $l=length($tmp_res);
#if($l<=3)
#{next;}
if(($l<3)&&($l!~/^[a-z|A-Z]/))
{next;}
if(exists($hash_st->{$tmp_res}))
{next;}
#原始词保留
#主题放大
$w=$tmp_res;
if($tmp_hash->{$w}>($list_len/10))
{next;}
else
{$tmp_hash->{$w}+=2;
$og->{$w}->{$w}=1;
}
$tmp_hash_seq->{$w}=$seq;
$seq++;
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
$tmp_hash->{$k}+=$v;
$tmp_hash_seq->{$k}=$seq;
$og->{$k}->{$w}=1;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k seq=$seq\n";
}
}
else
{
$tmp_hash->{$w}+=0.1;
$og->{$w}->{$w}=1;
$tmp_hash_seq->{$w}=$seq;
print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k seq=$seq\n";
}
}
return (\%{$tmp_hash},$nline,\%{$tmp_hash_seq}, \%{$og});
}
sub log2 {
my $n = shift;
return log($n)/log(2);
}
sub rewrite_gx
{
my $inp=$_;
my @re_res;
my @tl=split(" ",$inp);
my $otl;
my $has_re;
my $has_n=0;
my $no_has;
my $r;
print "scalar= ",scalar(@tl),"\n";
push(@re_res,$tl[0]);
for(1 ..(scalar(@tl)-1))
{
my $n=$_;
my $tmp=$tl[$_];
if(exists($rewrite_all->{$tmp}))
{
$has_re->{$tmp}=$rewrite_all->{$tmp};
$has_local->{$tmp}=$tmp;
$has_time->{$tmp}=$has_n;
$has_n++;
}
}
if($has_n<2){ print "rewrite skip $has_n $inp\n";return $inp;}
my $exists_tkl;
while(($k,$v)=each(%{$has_re}))
{
my @kl=split(" ",$v);
foreach my $tkl (@kl)
{
my $is_rewrite=1;
if(($rewrite_n[$tkl]>$has_n)||(exists($exists_tkl->{$tkl})))
{
next;
}
while(my($k1,$v1)=each(%{$rewrite_g[$tkl]}))
{
if(exists($has_re->{$k1}))
{}
else{$is_rewrite=0;last;}
}
if($is_rewrite==1)
{
$exists_tkl->{$tkl}=1;
push (@re_res,$rewrite_l[$tkl]);
while(my($k1,$v1)=each(%{$rewrite_g[$tkl]}))
{ $no_has->{$k1}=1;}
}
}
}
my $f=0;
foreach my $tt (@tl)
{
if($f==0){$f++;next;}
if(exists($no_has->{$tt}))
{
next;
}
push(@re_res,$tt);
}
$r=join(" ",@re_res);
return $r;
}
sub load_rewrite
{
my $rg=0;
my $name;
my $rg_hs;
open(FDre,"rewrite.txt");
while(<FDre>)
{
my $line=$_;
my @a=split("\t",$line);
my $f=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$rg++;
$name=$i;
if(length($name)<3)
{last;}
$rewrite_l[$rg]=$name;
my @rew=split("_",$name);
foreach my $t_rew (@rew)
{
$rewrite_n[$rg]++;
$rewrite_g[$rg]->{$t_rew}=1;
# $rg_hs[$rg]->{$t_rew}=$rg;
if(exists( $rewrite_all->{$t_rew}))
{
my $t_add=$rewrite_all->{$t_rew}." ".$rg;
$rewrite_all->{$t_rew}=$t_add;
}
else{$rewrite_all->{$t_rew}=$rg;}
}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
$hash->{$name}->{$av}=$bv;
}
}
}
}
}
sub deep_article_path
{
my $file=$_[0];
#chomp($file);
my @m_res;
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash;
open(FD_scws,"<$file");
my $key1=<FD_scws>;
chomp($key1);
my $innext=<FD_scws>;
my $key2=<FD_scws>;
chomp($key2);
print "----$key1|$key2\n";
if(exists($hash->{$key1}))
{
while(my ($nk,$nv)=each($hash->{$key1}))
{
if(exists($hash->{$key2}))
{
while(my($nk2,$nv2)=each($hash->{$key2}))
{
if($nk eq $nk2)
{
push(@m_res,"same $nk");
}
}
}
}
}
unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
@m_res;
}
sub dist
{
my @m_res;
my $la=$_[0];
my $lb=$_[1];
my $ina="0 $la";
my $inb="1 $lb";
#my ($my_hash,$n_line,$my_hash2_seq,$og2)=txt2arr3($in1,$exp_hash);
my ($my_hash)=txt2arr3($ina,$exp_hash);
my ($my_hash2)=txt2arr3($inb,$exp_hash);
$n=comp_hash($my_hash,$my_hash2);
$m_res[0]="0-1-$n";
@m_res;
}
sub comp_pin
{
my $a=$_[0];
my $b=$_[1];
print "a b $a $b\n";
foreach my $ap (@{$pinyinhash->{$a}})
{
foreach my $bp (@{$pinyinhash->{$b}})
{
if($bp eq $ap)
{
print "sw $a-$b-$ap-$bp\n";
return "1";}
}
}
print "dw $a-$b\n";
return "0";
}
#4——9支持 stop.txt 其中加了若干标点
# jieba分词 分割文件 bug修正
# jieba分词 实现多socket的全内存调用 待高并发测试
# 添加了client_qc.pl用来支持 分类引擎与切词引擎的数据交换
#添加了 若干词到 词典文件
#计划加入 每条 目标与待测文本 进行域向量修订(共现放缩) 域词典 数量为N
#4-12 修正词典中的标点，和去掉一些单字
#测试自然对数 作为衡量相似度的
# 4-14修正 reload 命令同时重启切词系统
#4-20 comp_hash加入相似特征的序列对比 当序列相近则认为相似度更近
#加入同义词词典
#4-23 单字支持做调整 修正了停用词表的bug 修正了文本中有----的bug
#4-29 修正输入 文章中有数字影响 权重的bug
#4-30加入post数据接口 加入 共现词组转义rewrite函数
#添加重复词语数量控制 为20个词中 最多计数一个
#添加了 各种品牌
#向量空间减少到60个
#加入目标语句的深度含义分析
#计划加入 实体路径的最大关联分析
#加入dist函数 直接分析2个词的距离