zhangbo-NLP-plugin_eng/lib/zhangbo/NLP/eng_org_hs.pl
#!/usr/bin/perl
use Socket;
use IO::Handle;
use IO::Select;
use IO::Socket::INET;
use lib "./";
use plugin_eng;
use POE qw(Component::Server::TCP);
#use JSON::XS;
use Clone qw(clone);
use Encode;
#$tt=seg_txt("我的西瓜很大");
#print $tt."222222222\n";
our @inputb_ar;
our @long_hs;
open(FDdic,"hs10.dic");
my $j=0;
while(<FDdic>)
{
my $nl=$_;
chomp($nl);
# print "$nl nl\n";
my @in;
@in=split("\t",$nl);
my $lin=seg_txt($in[1]);
# print "$lin in\n";
my ($my_hash2,$n_line,$my_hash2_seq,$og2)=txt2arr3("0 $lin",$exp_hash) ;
$inputb_ar[$j]=clone($my_hash2);
$long_hs[$j]=clone($my_hash2_seq);
$j++;
}
close FDdic;
our @inputdic=`cat hs10.dic`;
#use MongoDB;
use Smart::Comments;
#use Conn_mongo_jc;
use Data::Dumper;
#use IO::File;
#use Add_info;
our @rewrite_g;
our @rewrite_l;
our @rewrite_n;
our %rewrite_all;
our $rewrite_all;
#my $sel = IO::Select->new($socket); ##建立select对象
#system("./test_files2.py 1>/dev/null 2>/dev/null &");
#system("./post_server.pl &");
open(FDst,"stop.txt");
our $hash_st;
while(<FDst>)
{
my $line;
$line=$_;
chomp($line);
$hash_st->{$line}=1;
}
close FDst;
# http://poe.perl.org/?POE_Cookbook/TCP_Servers
# Include POE and POE::Component::Server::TCP.
our $hash=();
$hash_mn=();
my $hash_sc=();
#open(FD_sc, "dict.utf8.txt");
#open(FD_log, ">/tmp/engine.txt");
#while(<FD_sc>)
#{
#招收学员 13.87 8.50 n
# my $line=$_;
# if($line=~/(.*?)\t(.*?)\t(.*?)\t(.*)/)
# {
# $tt1=$1;
# $tt2=$2;
# $tt3=$3;
# $tt4=$4;
# $hash_sc->{$tt1}->{"i"}=$tt2;
# $hash_sc->{$tt1}->{"h"}=$tt3;
# $hash_sc->{$tt1}->{"a"}=$tt4;
#
# }
#}
#system("pwd");
#system("ls ../");
#print "@INC\n";
#print "$INC[1]\n";
#if(! -e "./word.txt")
#{print "dic file found ./\n";
#};
#if(! -e "$INC[1]/word.txt")
#{print "dic file found /\n";
#};
#if(! -e "../word.txt")
#{print "dic file found ../\n";
#};
#if(! -e "/home/lzj/shell2/word.txt")
#{print "dic file found /home/lzj/shell2/\n";
#};
open(FD, "sy.txt");
my $sytmp;
while(<FD>)
{
my $line=$_;
if($line=~/\#(.*)/)
{
$tsy=$1;
if(!exists($sytmp->{$tsy}))
{
$sytmp->{$tsy}=$tsy;
$eat=1;
# print "id=$tsy\n";
}
else
{
$eat=0;
}
}
elsif($eat==1)
{
chomp($line);
if(length($tsy)<=3)
{next;}
if(length($line)<=3)
{next;}
$hash->{$line}->{$tsy}=0.2;
# $hash->{$tsy}->{$line}=0.2;
# print "-$tsy-$line\n";
}
}
close FD;
open(FD, "word.txt");
my $tmp=();
while(<FD>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
chomp($line);
if($line=~/^W_C=(.*)/)
{
$tmp=$1;
#print $tmp,"\n";
$r=1;
}
elsif($line=~/^DEF=(.*)/)
{
$r=0;
$tmp_m=$1;
# print $tmp_m,"\n";
if(length($tmp)<=3)
{next;}
if(exists($hash_st->{$tmp}))
{next;}
my @arry=split('\|',$tmp_m);
foreach $tmps (@arry)
{
# print "-----------------$tmps ------------\n";
if($tmps=~/(.*?)\:/)
{
my $m=$1;
if(exists($hash_st->{$m}))
{next;}
# if($m eq "专")
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
# print "$tmp - $m-----\n";
if(exists($hash_sc->{$m}))
{
if($r==0)
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
}
else{ $hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*0.2;}
#$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.1;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
$r++;
}
elsif($tmps=~/(.*?)\}/)
{
my $m=$1;
# if($m eq "专")
#if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件"))
if(($m eq "专")||($m eq "功能词")||($m eq "人")||($m eq "事情")||($m eq "时间")||($m eq "特定")||($m eq "部件")||($m eq "地方"))
{next;}
if(exists($hash_st->{$m}))
{next;}
if($r==0)
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"};
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=1;
}
else
{
if(exists($hash_sc->{$m}))
{
$hash->{$tmp}->{$m}=$hash_sc->{$m}->{"h"}*(0.5);
if($hash->{$tmp}->{$m}<0.01){
$hash->{$tmp}->{$m}=0.2;}
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
else
{
$hash->{$tmp}->{$m}=1;
# print FD_log $hash->{$tmp}->{$m}," $m\n";
}
$hash_mn->{$tmp}+=0.5;
}
$r++;
}
}
}
}
open(FDb, "0babel_all_path_fin20.txt");
my $tmp=();
while(<FDb>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
$s=$line;
# $m=$2;
my $name;
@a=split("\t",$s);
my $dob=0;
#@b=split(" ",$m);
$f=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$name=$i;
if(length($name)<3)
{last;}
if(exists($hash->{$name}))
{
#print "$name\n";
$dob=1;
# last;
}
if(exists($hash_st->{$name}))
{last;}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
if(exists($hash->{$name}->{$av}))
{next;}
if($dob==1)
{
$hash->{$name}->{$av}=$bv*0.3;
}
else
{
$hash->{$name}->{$av}=$bv;
}
}
}
}
#foreach $i (@b)
#{
# $hash->{$name}->{$i}+=0.01;
#}
}
close(FDb);
sub reloaddic
{
open(FDst,"stop.txt");
our $hash_st;
while(<FDst>)
{
my $line;
$line=$_;
chomp($line);
$hash_st->{$line}=1;
}
close FDst;
load_dic("babel_add.txt",60,0.5);
load_dic("名词.txt",20,0.5);
load_dic("动词.txt",10,0.5);
load_dic("zw.txt",10,0.2);
}
sub load_dic
{
open(FDb, "$_[0]");
my $lsize=$_[1];
my $zoomin=0.5;
if($_[2]>0)
{$zoomin=$_[2];}
my $tmp=();
while(<FDb>)
{
#DEF={Unit|单位:host={information|信息:belong={computer|电脑}}}
#DEF={tool|用具:modifier={able|能:scope={bring|携带:content={$}}},{listen|听:content={music|音乐},instrument={~}}}
$line=$_;
$s=$line;
# $m=$2;
my $name;
@a=split("\t",$s);
my $dob=0;
#@b=split(" ",$m);
$f=0;
my $ccn=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$name=$i;
#if(exists($hash->{$name}))
#{
#print "$name\n";
# $dob=1;
# last;
#}
# if(exists($hash_st->{$name}))
# {last;}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
#覆盖
if(exists($hash_st->{$av}))
{next;}
if($ccn>$lsize)
{last;}
$ccn++;
$hash->{$name}->{$av}=$bv*$zoomin;
# print "$name $av $bv\n";
}
}
}
#foreach $i (@b)
#{
# $hash->{$name}->{$i}+=0.01;
#}
}
close (FDb);
print "load $_[0] ok\n";
}
#close FD_log;
load_dic("base_total.txt",15,8);
#load_dic("base_total.txt",15,8);
#load_dic("babel_add.txt",15,1);
#load_dic("more.txt",15,1);
load_dic("名词.txt",15,1);
#load_dic("动词.txt",10,0.5);
#load_dic("zw.txt",10,0.2);
#reloaddic();
#load_rewrite();
# print Data::Dumper->Dump([%{$rewrite_g[1]}]);
#print Data::Dumper->Dump([%{$hash}]);
#print "-----\n";
# print Data::Dumper->Dump([%{$rewrite_all}]);
#print "-----\n";
print "server start\n";
my $n=0;
#our $f=0;
my $noshudown=0;
my $online=0;
my $taskmsg;
POE::Component::Server::TCP->new(
Alias => "echo_server",
Port => 11271,
ClientInput => sub {
my ($session, $heap, $input) = @_[SESSION, HEAP, ARG0];
# print "Session ", $session->ID(), " got input: ".$input."\n";
my $msid= $session->ID();
#print "Session ", $session->ID(), " got input: ".length($input)."\n";
my $d;
my $d_seq;
#open(FD_log,">/tmp/engine.log");
#print FD_log "$input\n";
#system("cp $input /tmp/");
#print "input |$input|",length($input),"\n";
my @res;
#my $online=0;
if(length($input)<3){
$_[KERNEL]->yield("shutdown");
return;
}
if($input=~/(.*) showtop (\d+)/)
#@res=deep_article($input);
{
@res=search_article($1,"showtop");
#system("cat $1");
}
elsif($input eq "<START noseg>")
{
$taskmsg->{"online"}->{$msid}=1;
$taskmsg->{"noshudown"}->{$msid}="yes";
$taskmsg->{"seg input"}->{$msid}="no";
}
elsif($input eq "<START>")
{
$taskmsg->{"online"}->{$msid}=1;
$taskmsg->{"noshudown"}->{$msid}="yes";
$taskmsg->{"seg input"}->{$msid}="yes";
#delete($taskmsg->{$msid});
}
elsif($input eq "<STOP>")
{
#print "stop input ".@taskmsg."\n";
$taskmsg->{"online"}->{$msid}=0;
$taskmsg->{"noshudown"}->{$msid}="no";
@res=search_article(\@{$taskmsg->{$msid}},"online",$taskmsg->{"seg input"}->{$msid});
$heap->{client}->put(@res);
delete($taskmsg->{$msid});
$_[KERNEL]->yield("shutdown");
return;
}
elsif(($taskmsg->{"online"}->{$msid}==1)&&($taskmsg->{"noshudown"}->{$msid}=="yes"))
{
push (@{$taskmsg->{$msid}},$input);
#print "push ".$input."\n";
}
else
{
@res=search_article($input,"","yes");
}
#print FD_log @res;
#close(FD_log);
#print scalar(@res)," num\n";
# $heap->{client}->put(@res);
#$session->yield("shutdown");
if($taskmsg->{"noshudown"}->{$msid} eq "yes")
{
}
else
{
$heap->{client}->put(@res);
$_[KERNEL]->yield("shutdown");
return;
}
},
ClientDisconnected => sub {
#print "Client disconnected\n"; # log it
}
#, ClientFlushed => sub {
# my $data_source = $_[HEAP]{file_handle};
# my $read_count = sysread($data_source, my $buffer = "", 65536);
# if ($read_count) {
# $_[HEAP]{client}->put($buffer);
# }
# else {
# print FD_log "------------------error\n";
# $_[KERNEL]->yield("shutdown");
# }
#}
);
POE::Kernel->run;
#$file=$ARGV[0];
sub search_article
{
my @input_ar;
my @input_ar2;
my $file;
my $showtop=0;
my $ag1=$_[0];
my $ag2=$_[1];
my $ag3=$_[2];
print "file= ".$ag1."----".$ag2."\n";
if($ag2 eq "showtop")
{
$file=$ag1;
$showtop=1;
print "showtop 1 $file\n";
print "showtop 2 $showtop\n";
#chomp($file);
@input_ar=`cat $file`;
my $start_c=0;
my $sbuf;
my $tn=0;
}
my $f=0;
my $n=0;
my $thash=();
my @thash;
my %my_hash;
my %my_hash2;
my %my_hash2_seq;
my %thash;
my $j;
my $input_dis;
my %input_dis;
#my $rand2=rand();
#open(FD_hash,">$file.txt");
#print "openfile $file\n";
#open(FD_arr,"<$file");
my $first=0;
my $rand=rand();
#open(FD_aro,">/tmp/$rand.sctxt");
my $j=1;
my @inputa;
my @inpall;
my @inputb;
#while(<FD_arr>)
#print FD_log $j,"----------\n";
#print $j,"----------\n";
#print FD_log "open /tmp/$rand.sctxt2 \n";
my $disthash;
my @disthash;
my $dist_seq_hash;
my @dist_seq_hash;
#while(<FD_scws>)
my $kk=0;
my $in;
$in=$input_ar[0];
#print "$in= ".$inpall[$fff].".ininini\n";
# my $in=$inpall[$fff];
#print FD_log "start \n";
#print FD_log "line ttt $kk $in \n";
chomp($in);
$in=seg_txt($in);
$in="0 $in";
print "00000 $in\n";
my ($d,$c,$long_hasha,$og)=txt2arr_input($in,$exp_hash);
$disthash[$kk]=clone($d);
$disthashog[$kk]=clone($og);
$dist_seq_hash[$kk]=clone($long_hasha);
$kk++;
#print Data::Dumper->Dump([%{$d}]);
# %my_hash2=txt2arr($input2);
# $n=comp_hash(\%my_hash,\%my_hash2);
for( 0 .. scalar(@inputb_ar)-1)
{
my $nown=$_;
my $n_line=$nown;
# $in=rewrite_gx($in);
# print "in2 == $in\n";
# my ($my_hash2,$n_line)=txt2arr3($in,$exp_hash);
# $in=seg_txt($in);
#print "in2 $in\n";
# my ($my_hash2,$n_line,$long_hash2,$og2)=txt2arr3($in,$exp_hash);
my $samelist;
my $n;
($n,$samelist)=comp_hash($disthash[0],$inputb_ar[$nown],$dist_seq_hash[0],$long_hs[$nown],$disthashog[$nnn],$og2);
#if($n>0)
# {
# print $samelist."$n $n_line $inputdic[$n_line] ]]]\n";
# }
# while (($k,$v)=each(%{$disthash[0]}))
# {
# print "a $k,$v\t";
# }
# while (($k,$v)=each(%{$inputb_ar[$nown]}))
# {
# print "b $k,$v\t";
# }
# print FD_hash "\n";
#($n,$samelist)=comp_hash($disthash[$nnn],$my_hash2);
# my $shnn=$nnn+1;
#print "totle $samelist $n_line=$n $inputdic[$n_line]\n";
#exit;
#print FD_log "$inputa[$nnn]\n$inputb[$n_line-1]\n";
$thash[0]->{$n_line}=$n;
# print "$n_line $n\n";
#if($showtop == 1)
# {
# chomp($in);
#print "input_dis |$nnn| $n_line| $samelist\n";
$input_dis->{"1-$n_line"}=$samelist;
#print FD_log "inputdis $n_line $input_dis->{$n_line}\n";
# }
}
#unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
# close(FD_arr);
my @m_res;
if($showtop ==1 )
{
for(0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
my @key =sort {$thash[$nnn]->{$b} <=> $thash[$nnn]->{$a}} keys %{$thash[$nnn]};
#print @key;
my $o_n;
my $toptmp=0;
my $jjj=0;
foreach $i (@key)
{
# if( $thash[$nnn]->{$i}>0)
# {
if($jjj>5)
{last;}
$jjj++;
my $shnnn=$nnn+1;
print "put input_dis |$nnn| $i| ".$input_dis->{"$nnn-$i"}."\n";
push (@m_res,"1-$i-".$thash[$nnn]->{$i}."-".$inputdic[$i]."-".$input_dis->{"1-$i"}." ");
#push (@m_res,"0-1-".$thash[0]->{"1"}."-".$input_dis->{"1"});
#print FD_log "inputdis show $i $input_dis->{$i}\n";
#push (@m_res,"$nnn-$i-".$thash[$nnn]->{$i}.$input_ar[$i]);
# $o_n++;
# }
# $toptmp++;
}
}
}
else
{
for(0 .. (scalar(@disthash)-1))
{
my $nnn=$_;
my @key =sort {$thash[$nnn]->{$b} <=> $thash[$nnn]->{$a}} keys %{$thash[$nnn]};
#print @key;
my $o_n;
foreach $i (@key)
{
# if( $thash[$nnn]->{$i}>0)
# {
my $shnnn=$nnn+1;
push (@m_res,"$shnnn-$i-".$thash[$nnn]->{$i});
#push (@m_res,"0-1-".$thash[0]->{"1"});
#push (@m_res,"$nnn-$i-".$thash[$nnn]->{$i}.$input_ar[$i]);
# $o_n++;
# }
}
}
}
# print FD_log "return ",scalar(@m_res),"\n";
@m_res;
}
# }
# );
#$poe_kernel->run();
sub txt2arr
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
# chomp($input_t);
my @list=split(" ",$input_t);
#print "txt2arr input =$input_t\n";
#$rand=rand();
#open(FD_r,">/tmp/$rand");
#print FD_log "$input_t\n";
#close(FD_r);
#$dir="/home/wyb/stanford-parser-2012-07-09";
@res=();
#@res=`$dir/parser-10.pl /tmp/$rand 0 50`;
#$part=0;
#unlink("/tmp/$rand");
#print @res;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
if(exists($hash_st->{$tmp_res}))
{next;}
my $l=length($tmp_res);
# if($l<=3)
# {
#if($hash_sc->{$tmp_res}->{'a'} !~/n|v|a/)
#{
#print FD_log "seek $tmp_res\n";
#next;}
if($l<3)
{next;}
#主题放大
$tmp_hash->{$tmp_res}++;
# }
# print $tmp_res,"---",length($tmp_res),"\n";
#if (($part==1)&&($tmp_res=~/(.*?),(.*)/))
#{
#print "切词1=$1,2=$2\n";
my $w=$tmp_res;
# my $value=$2;
# if(exists($exp_h->{$w}))
# {
# print "$w\n";
# next;}
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
if(exists($hash_st->{$k}))
{next;}
$tmp_hash->{$k}+=$v;;
#$tmp_hash->{$k}=$v;
#$tmp_hash->{$k}=$hash_sc->{$w}->{'h'};
# print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k\n";
#正常词 交 转换0.00--0.00 相互
#if(exists($sytmp->{$k}))
#{
# print "归一 $k ",$sytmp->{$k},"\n";
# $k=$sytmp->{$k};
#}
# if($value>20)
# $heap->{client}->put("$k $v\n");
}
}
}
return (\%{$tmp_hash},$nline);
}
#print "output = $res\n";
# $heap->{client}->stop();
# $session->stop();
# Start the server.
sub comp_hash
{
my $h1=@_[0];
my $h2=@_[1];
my $h3=@_[2];
my $h4=@_[3];
my $org1=@_[4];
my $org2=@_[5];
my $samel;
#while(($k,$v)=each(%{$h3}))
# {
#print "h3 show $k, $v\n";
# }
#while(($k,$v)=each(%{$h4}))
# {
#print "h4 show $k, $v\n";
# }
my $akv=0;
my $all;
my $all_l;
my $an_all;
my %all;
my $vall=0.001;
my $vall2=0.001;
my $k,$v;
while(($k,$v)=each(%{$h1}))
{
$all->{$k}++;
if($v>0)
{
$an_all++;
#$vall++;
$vall+=$v;
}
# print " hash 1 k $k v $v\n";
}
my $k,$v;
while(($k,$v)=each(%{$h2}))
{
$all->{$k}++;
if($v>0)
{
$an_all++;
#$vall2++;
$vall2+=$v;
}
# print " hash 2 k $k v $v\n";
}
my $k,$v;
my $samelist;
my $std_n;
while(($k,$v)=each(%{$all}))
{
if($v >= 2)
{
# my $sum;
#if($h1->{$k}>$h2->{$k})
#{$sum=$h2->{$k};}
#else
#{$sum=$h1->{$k};}
#$kv=$sum/($vall+$vall2);
if($h1->{$k}<0)
{next;}
$an_all=$an_all-2;
# 平均算法 适用于多词找文 对词典中噪音有包容性 #$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
#$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2);
#$kv2=(($h1->{$k}) + ($h2->{$k}/$vall2))/2;
#$kv3=($kv+$kv2)/2;
# log2算法 适用于文找文 $kv=($h1->{$k}+$h2->{$k})/($vall+$vall2+log2($an_all));
#$kv=($h1->{$k}+$h2->{$k})/($vall+$vall2+log2($an_all)/2);
#$kv3=$kv;
#$kv2=((($h1->{$k})/$vall) + ($h2->{$k}/$vall2))/2;
#if($h1->{$k}<$h2->{$k})
#{
#}
$akv+=$kv+0.15;
# if($akv<0.2){$akv+=0.1;}
# else{$akv+=$kv;}
if($akv>=0.9){ $akv=0.9;}
# $akv+=$kv3;
# print FD_log "double $k $v kv sum= $h1->{$k}+$h2->{$k} ($org1->{$k}, $org2->{$k}) kv= $kv val1= $vall val2= $vall2 kv2=$kv2 kv3 =$kv3 std_n=$std_n\n";
# print "double $akv $k $v kv sum= $h1->{$k}+$h2->{$k} ($org1->{$k}, $org2->{$k}) kv= $kv val1= $vall val2= $vall2 kv2=$kv2 kv3 =$kv3 std_n=$std_n\n";
$samel .=" $k";
# print FD_log "相似语义 $k 来自前句 from \n";
# while(($kc,$vc)=each(%{$org1->{$k}}))
# {
# print FD_log "$kc,";
# }
#print FD_log " 来自后句 ";
#if (exists(%{$org2->{$k}}))
#{
# while(($kc,$vc)=each(%{$org2->{$k}}))
#{
# print FD_log "$kc,";
#}
#print "\n";
# }
#$samelist->{$k}=$h3->{$k};
$std_n++;
}
else
{
$akv=$akv-0.01;}
}
my $k,$v;
my $vall_l;
my $vall2_l;
my $all_l;
my $all2_l;
my $bkv;
while(($k,$v)=each(%{$h3}))
{
$all_l->{$k}++;
if($v>0)
{
$an_all++;
$vall_l+=$v;
}
}
while(($k,$v)=each(%{$h4}))
{
$all_l->{$k}++;
if($v>0)
{
$vall2_l+=$v;
}
}
while(($k,$v)=each(%{$all_l}))
{
if($v >= 2)
{
#if(exists($h1->{$k}))
#{$bkv+=$v*0.01;}
$kv=($h3->{$k}+$h4->{$k})/($vall_l+$vall2_l);
# if($bkv>0.9){}
if($bkv<0.1){$bkv+=0.001;}
else{$bkv+=$kv*0.01;}
# print "double bkv $bkv $k $v kv sum= $h1->{$k}+$h2->{$k} ($org1->{$k}, $org2->{$k}) kv= $kv val1= $vall val2= $vall2 kv2=$kv2 kv3 =$kv3 std_n=$std_n\n";
}
}
$akv=$akv+$bkv;
if($akv>0.9){$akv=0.9;}
#序列相似评估
#---------------------------------------
if(0)
#if(($std_n>0)&&($std_n !=1))
{
# make samelist ->std test_seq->obj_seq;
my @key =sort {$samelist->{$b} <=> $samelist->{$a}} keys %{$samelist};
#print @key;
my $std_seq;
my $t_n=1;
my $test_seq;
my $last_tmp;
my $my_fi=0;
my $std_sq;
foreach $i (@key)
{
if(exists($std_sq->{$samelist->{$i}}))
{
$std_seq->{$i}=($std_n-$t_n+1);
$test_seq->{$i}=$h4->{$i};
# print FD_log "def test_seq $i =$h4->{$i}\n";
# print FD_log "std_seq last $i seq =$std_seq->{$i} h3 $h3->{$i} std_n=$std_n\n";
}
else
{
$t_n++;
$std_sq->{$samelist->{$i}}=1;
$std_seq->{$i}=($std_n-$t_n+1);
# print FD_log "std_seq $i seq=$std_seq->{$i}\n";
# print FD_log "def test_seq $i =$h4->{$i}\n";
$test_seq->{$i}=$h4->{$i};
}
}
my @key =sort {$test_seq->{$b} <=> $test_seq->{$a}} keys %{$test_seq};
my $t_n=1;
my $last_tmp;
my $obj_tmp;
my $test_sq;
foreach $i (@key)
{
# print FD_log "$i \"$last_tmp\" == \"$test_seq->{$i}\" ??\n";
# if($last_tmp eq $test_seq->{$i})
if(exists($test_sq->{$test_seq->{$i}}))
{
$obj_seq->{$i}=($std_n-$t_n+1);
# print FD_log "test_seq last $i seq= $obj_seq->{$i} std_n=$std_n\n";
}
else
{
$t_n++;
$test_sq->{$test_seq->{$i}}=1;
$obj_seq->{$i}=($std_n-$t_n+1);
}
# test_seq 受体拮抗剂 last_tmp = 26 25
# print FD_log "test_seq $i seq= $obj_seq->{$i}\n";
}
my $diff_sum=0;
while(($k,$v)=each(%{$std_seq}))
{
$diff_sum=$diff_sum+abs($v-$obj_seq->{$k});
# print FD_log "diff_sum $diff_sum $k $v $obj_seq->{$k}\n";
}
my $sam_rate=1-($diff_sum/(($std_n-1)*($std_n-1)));
my $diff_n=$akv*(1-$sam_rate)*0.15;
$akv=$akv-$diff_n;
#print FD_log "seq diff sam_rate $sam_rate akv $akv diff_n $diff_n\n";
}
#-----------------------------------------------
# 异议评估 说多了远的，深意不在这里
# my $maxd=log2($an_all+1);
# if($maxd>100)
# {$akv=$akv*0.618;}
# elsif($maxd==0)
# {
# }
# elsif($maxd>1)
# {
# $akv=$akv*(0.8+0.2/($maxd+1));
# }
# print "---------------\n";
return($akv,$samel);
}
sub ndate
{
($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time());
$year=~s/1(.*)/20$1/;
$mon+=1;
if($mon=~/^\d$/)
{
$mon="0"."$mon";
}
if($mday=~/^\d$/)
{
$mday="0"."$mday";
}
if($hour=~/^\d$/)
{
$hour="0"."$hour";
}
if($min=~/^\d$/)
{
$min="0"."$min";
}
if($sec=~/^\d$/)
{
$sec="0"."$sec";
}
my $res=$year."-".$mon."-".$mday." ".$hour.":"."$min".":"."$sec";
#print $res,"\n";
$res;
}
sub txt2arr_one
{
my $w=$_[0];
my $exp_h=$_[1];
$tmp_hash=();
$rand=rand();
@res=();
$part=0;
#print @res;
my @w_arr=split(",",$w);
foreach my $w_tmp (@w_arr)
{
if($w_tmp=~/(.*?) (.*)/)
{
$w=$1;
$vh=$2;
#print "$w -- $vh\n";
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
if($v==0)
{
$v=1;
}
$tmp_hash->{$k}=$vh;
}
}
$tmp_hash->{$w}=$w;
}
else
{
print "input error\n";
}
}
# while(($k,$v)=each(%{$tmp_hash}))
#
# {
#print "meaning $k $v\n";
# }
# }
return (%{$tmp_hash});
}
sub txt2arr_input
{
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
my $tmp_sequ;
my $sequ=0;
my @list=split(" ",$input_t);
#print FD_log "$input_t\n";
@res=();
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $tmp_first=0;
my $og;
my $long_hash;
my $linst_len;
$list_len=scalar(@list);
for (0 .. ($list_len-1))
#foreach $tmp_res (@list)
{
$nown=$_;
$tmp_res=$list[$_];
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
my $l=length($tmp_res);
#if(($l<4)&&($l!~/^[a-z|A-Z]/))
if($l<4)
#单子不处理 if(($l<)&&($l!~/^[a-z|A-Z]/))
{next;}
if(exists($hash_st->{$tmp_res}))
{next;}
if($tmp_res=~/^\d/)
{next;}
if($tmp_res=~/^-/)
{next;}
#原始词保留
#主题放大
my $zs=1;
#-----------------------------------------------------------------------
$w=$tmp_res;
if(($list[$nown+1]=~/^\d/)&&(length($input_t)<10))
{
$tmp_hash->{$w}+=2*$list[$nown+1];
$zs=$list[$nown+1];
}
elsif(($list[$nown+1]=~/^-/))
{
$tmp_hash->{$w}-=2*$list[$nown+2];
$zs=-$list[$nown+2];
}
else
{
#控制重复词语数量
# if($tmp_hash->{$w}>($list_len/10))
#if($tmp_hash->{$w}>($list_len/10))
# {next;}
# else
$tmp_hash->{$w}+=2;
$og->{$w}->{$w}=1;
$tmp_sequ->{$w}=$sequ;
$sequ++;
}
#----------------------------------------------------------
##序列
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
$long_hash->{$k}+=0.1;
# $og->{$k}->{$w}=1;
# $tmp_sequ->{$k}=$sequ;
# print "正常词 $w 转换","--",$k," k=$k sequ=$sequ\n";
}
}
else
{
$tmp_hash->{$w}+=0.01;
# $og->{$w}->{$w}=1;
# $tmp_sequ->{$k}=$sequ;
# print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k},"k=$k sequ=$sequ\n";
}
}
## $long_hash
return (\%{$tmp_hash},$nline,\%{$long_hash},\%{$og});
}
sub txt2arr3
{
#需要改进为 先判断文本域 再根据域 重写词向量值 域向量集 可以用clone覆盖已有向量
#例如黑话库 的词依存 规则
my $input_t=$_[0];
my $exp_h=$_[1];
my $tmp_hash=();
my $tmp_hash_seq=();
my @list=split(" ",$input_t);
#print FD_log "$input_t\n";
@res=();
my $og;
my $n_a=0;
my $n_b=0;
my $v_a=0;
my $v_b=0;
my $n1;
my $n2;
my $v1;
my $v2;
my $nline;
my $seq=0;
my $tmp_first=0;
my $list_len;
$list_len=scalar(@list);
my $long_hash;
foreach $tmp_res (@list)
{
if ($tmp_first==0)
{
$nline=$tmp_res;
$tmp_first++;
next;
}
my $l=length($tmp_res);
#if($l<=3)
#{next;}
if($l<3)
#if(($l<3)&&($l!~/^[a-z|A-Z]/))
{next;}
if(exists($hash_st->{$tmp_res}))
{next;}
#原始词保留
#主题放大
$w=$tmp_res;
if($tmp_hash->{$w}>($list_len/20))
{next;}
else
{$tmp_hash->{$w}+=2;
# $og->{$w}->{$w}=1;
}
#$tmp_hash_seq->{$w}=$seq;
#$seq++;
if(exists($hash->{$w}))
{
my $t_n=0;
while(($k,$v)=each(%{$hash->{$w}}))
{
$long_hash->{$k}+=$v;
# $tmp_hash_seq->{$k}=$seq;
# $og->{$k}->{$w}=1;
# print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k seq=$seq\n";
}
}
else
{
# if($leng_hash->{$k}<0.4)
# {
$tmp_hash->{$w}+=0.1;
# $leng_hash->{$k}+=0.1
# }
# $og->{$w}->{$w}=1;
# $tmp_hash_seq->{$w}=$seq;
# print FD_log "正常词 $w 转换",$hash_sc->{$w}->{'h'},"--",$tmp_hash->{$k}," $k seq=$seq\n";
}
}
return (\%{$tmp_hash},$nline,\%{$long_hash}, \%{$og});
}
sub log2 {
my $n = shift;
return log($n)/log(2);
}
sub rewrite_gx
{
my $inp=$_;
my @re_res;
my @tl=split(" ",$inp);
my $otl;
my $has_re;
my $has_n=0;
my $no_has;
my $r;
print "scalar= ",scalar(@tl),"\n";
push(@re_res,$tl[0]);
for(1 ..(scalar(@tl)-1))
{
my $n=$_;
my $tmp=$tl[$_];
if(exists($rewrite_all->{$tmp}))
{
$has_re->{$tmp}=$rewrite_all->{$tmp};
$has_local->{$tmp}=$tmp;
$has_time->{$tmp}=$has_n;
$has_n++;
}
}
if($has_n<2){ print "rewrite skip $has_n $inp\n";return $inp;}
my $exists_tkl;
while(($k,$v)=each(%{$has_re}))
{
my @kl=split(" ",$v);
foreach my $tkl (@kl)
{
my $is_rewrite=1;
if(($rewrite_n[$tkl]>$has_n)||(exists($exists_tkl->{$tkl})))
{
next;
}
while(my($k1,$v1)=each(%{$rewrite_g[$tkl]}))
{
if(exists($has_re->{$k1}))
{}
else{$is_rewrite=0;last;}
}
if($is_rewrite==1)
{
$exists_tkl->{$tkl}=1;
push (@re_res,$rewrite_l[$tkl]);
while(my($k1,$v1)=each(%{$rewrite_g[$tkl]}))
{ $no_has->{$k1}=1;}
}
}
}
my $f=0;
foreach my $tt (@tl)
{
if($f==0){$f++;next;}
if(exists($no_has->{$tt}))
{
next;
}
push(@re_res,$tt);
}
$r=join(" ",@re_res);
return $r;
}
sub load_rewrite
{
my $rg=0;
my $name;
my $rg_hs;
open(FDre,"rewrite.txt");
while(<FDre>)
{
my $line=$_;
my @a=split("\t",$line);
my $f=0;
foreach $i (@a)
{
if($f==0)
{
$f++;
$rg++;
$name=$i;
if(length($name)<3)
{last;}
$rewrite_l[$rg]=$name;
my @rew=split("_",$name);
foreach my $t_rew (@rew)
{
$rewrite_n[$rg]++;
$rewrite_g[$rg]->{$t_rew}=1;
# $rg_hs[$rg]->{$t_rew}=$rg;
if(exists( $rewrite_all->{$t_rew}))
{
my $t_add=$rewrite_all->{$t_rew}." ".$rg;
$rewrite_all->{$t_rew}=$t_add;
}
else{$rewrite_all->{$t_rew}=$rg;}
}
}
else
{
if($i=~/(.*?)\:(.*)/)
{
my $av=$1;
my $bv=$2;
$hash->{$name}->{$av}=$bv;
}
}
}
}
}
sub deep_article_path
{
my $file=$_[0];
#chomp($file);
my @m_res;
my $f=0;
my $n=0;
my $kk;
my $thash=();
my @thash;
open(FD_scws,"<$file");
my $key1=<FD_scws>;
chomp($key1);
my $innext=<FD_scws>;
my $key2=<FD_scws>;
chomp($key2);
print "----$key1|$key2\n";
if(exists($hash->{$key1}))
{
while(my ($nk,$nv)=each(%{$hash->{$key1}}))
{
if(exists($hash->{$key2}))
{
while(my($nk2,$nv2)=each(%{$hash->{$key2}}))
{
if($nk eq $nk2)
{
push(@m_res,"same $nk");
}
}
}
}
}
unlink("/tmp/$rand.sctxt2");
#close(FD_hash);
@m_res;
}
sub dist
{
my @m_res;
my $la=$_[0];
my $lb=$_[1];
my $seg=$_[2];
if($seg eq "qc")
{
#my $a=`./client_qc_seg.pl $la`;
#my $b=`./client_qc_seg.pl $lb`;
my $a=seg_txt($la);
my $b=seg_txt($lb);
chomp($a);
chomp($b);
my $ina="0 $a";
my $inb="1 $b";
print "$ina,$inb inainb\n";
#$ina=decode("utf-8",$ina);
#$inb=decode("utf-8",$inb);
my ($my_hash)=txt2arr3($ina,$exp_hash);
my ($my_hash2)=txt2arr3($inb,$exp_hash);
$n=comp_hash($my_hash,$my_hash2);
$m_res[0]="0-1-$n";
}
else
{
my $ina="0 $la";
my $inb="1 $lb";
#my ($my_hash,$n_line,$my_hash2_seq,$og2)=txt2arr3($in1,$exp_hash);
my ($my_hash)=txt2arr3($ina,$exp_hash);
my ($my_hash2)=txt2arr3($inb,$exp_hash);
$n=comp_hash($my_hash,$my_hash2);
$m_res[0]="0-1-$n";
}
@m_res;
}
sub comp_pin
{
my $a=$_[0];
my $b=$_[1];
print "a b $a $b\n";
foreach my $ap (@{$pinyinhash->{$a}})
{
foreach my $bp (@{$pinyinhash->{$b}})
{
if($bp eq $ap)
{
print "sw $a-$b-$ap-$bp\n";
return "1";}
}
}
print "dw $a-$b\n";
return "0";
}
sub seg_txt
{
my $txt=$_[0];
my $socket = IO::Socket::INET->new(
PeerAddr => "127.0.0.1",
PeerPort => "11229",
Type => SOCK_STREAM,
Proto => "tcp",
)
or die "Can not create socket connect.$@";
$socket->autoflush(1);
$sel = IO::Select->new($socket);
$socket->send("$txt\n",0); ##发送消息至服务器端。
my $i;
# print "$i socket\n";
while (my @ready = $sel->can_read) { ##等待服务端返回的消息
foreach my $fh (@ready) {
if ($fh == $socket) {
while (<$fh>) {
$i=$_;
#print "read====$i==================\n";
}
$sel->remove($fh);
close $fh;
}
}
}
$i;
}
#4——9支持 stop.txt 其中加了若干标点
# jieba分词 分割文件 bug修正
# jieba分词 实现多socket的全内存调用 待高并发测试
# 添加了client_qc.pl用来支持 分类引擎与切词引擎的数据交换
#添加了 若干词到 词典文件
#计划加入 每条 目标与待测文本 进行域向量修订(共现放缩) 域词典 数量为N
#4-12 修正词典中的标点，和去掉一些单字
#测试自然对数 作为衡量相似度的
# 4-14修正 reload 命令同时重启切词系统
#4-20 comp_hash加入相似特征的序列对比 当序列相近则认为相似度更近
#加入同义词词典
#4-23 单字支持做调整 修正了停用词表的bug 修正了文本中有----的bug
#4-29 修正输入 文章中有数字影响 权重的bug
#4-30加入post数据接口 加入 共现词组转义rewrite函数
#添加重复词语数量控制 为20个词中 最多计数一个
#添加了 各种品牌
#向量空间减少到60个
#加入目标语句的深度含义分析
#计划加入 实体路径的最大关联分析
#加入dist函数 直接分析2个词的距离
#支持模糊推理
#调整扩展语义权重不大于2