wiki上关于KShingling Algorithm（w-shingling）的说明：
http://en.wikipedia.org/wiki/W-shingling
摘要：
In natural language processing a w-shingling is a set of unique "shingles"—contiguous subsequences of tokens in a document—that can be used to gauge the similarity of two documents. The w denotes the number of tokens in each shingle in the set.
The document, "a rose is a rose is a rose" can be tokenized as follows:
- (a,rose,is,a,rose,is,a,rose)
The set of all contiguous sequences of 4 tokens (N-grams, here: 4-grams) is
- { (a,rose,is,a), (rose,is,a,rose), (is,a,rose,is), (a,rose,is,a), (rose,is,a,rose) } = { (a,rose,is,a), (rose,is,a,rose), (is,a,rose,is) }
我理解的此算法，是把每段文本都像上述分解后，统计两段文本的合集b，再统计交集a，用a/b得到相似度。
写得有些复杂：
1 package bigproject2;
2
3 import javax.swing.JOptionPane;
4
5 public class union {
6 //求子集
7 public String[] ziji(String str)
8 {
9 char[] ch=str.toCharArray();
10 int c=0;
11 for(int i=0;i<ch.length;i++)
12 {
13 if(ch[i]==‘ ‘)
14 c++;
15 }
16 //建立单词数组
17 String[] strt=new String[c+1];
18 for(int i=0;i<c+1;i++)
19 strt[i]="";
20 int h=0;
21 for(int i=0;i<c+1;i++)
22 {
23 for(int j=h;j<ch.length;j++)
24 {
25 if(ch[j]==‘ ‘)
26 {
27 h=j+1;
28 break;
29 }
30 else strt[i]+=ch[j];
31 }
32 }
33 return strt;
34 }
35 //按k分，并去掉重复子集。
36 public String[] cut(String[] str,int k) throws MyException{
37 if(str.length<k)
38 throw new MyException("单词数少于"+k+"，无法进行计算！");
39 String[] t=new String[str.length-k+1];
40 for(int i=0;i<str.length-k+1;i++)
41 t[i]="";
42 int h=0,m=0;
43 for(;h<str.length-k+1;h++)
44 {
45 for(int i=m;i<m+k;i++)
46 t[h]+=str[i];
47 m++;
48 }
49 //去掉重复部分
50 int merge=0;
51 for(int i=0;i<t.length-1;i++)
52 {
53 if(t[i].equals("")) break;
54 for(int j=i+1;j<t.length;j++)
55 {
56 if(t[i].equals(t[j]))
57 {
58 merge++;
59 int y=j;
60 for(;y<t.length-1;y++)
61 {
62 t[y]=t[y+1];
63 }
64 t[y]="";
65 }
66 }
67 }
68 String[] fin=new String[t.length-merge];
69 for(int i=0;i<t.length-merge;i++)
70 fin[i]=t[i];
71 return fin;
72 }
73 public class MyException extends Exception{
74 public MyException(String str){
75 JOptionPane.showMessageDialog(null, str,"警告", JOptionPane.INFORMATION_MESSAGE);
76 }
77 }
78 //求两字符串数组合集个数。
79 public int heji(String[] a,String[] b){
80 int count=a.length+b.length;
81 for(int i=0;i<a.length;i++)
82 {
83 for(int j=0;j<b.length;j++)
84 {
85 if(a[i].equals(b[j]))
86 count--;
87 }
88 }
89 return count;
90 }
91 //求两字符串数组交集个数。
92 public int jiaoji(String[] a,String[] b){
93 int count=0;
94 for(int i=0;i<a.length;i++)
95 {
96 for(int j=0;j<b.length;j++)
97 {
98 if(a[i].equals(b[j]))
99 count++;
100 }
101 }
102 return count;
103 }
104
105 }
1 package bigproject2;
2
3
4 public class KShinglingAlgorithm extends union{
5 private String text1,text2;
6 public String getText1()
7 {
8 return text1;
9 }
10 public String getText2()
11 {
12 return text2;
13 }
14 public void setText1(String text1)
15 {
16 this.text1=text1;
17 }
18 public void setText2(String text2)
19 {
20 this.text2=text2;
21 }
22
23 public float getSimilarity(int k)
24 {
25 union a=new union();
26 String[] t1=a.ziji(this.text1);
27 String[] t2=a.ziji(this.text2);
28 String[] t1t,t2t;
29 try{
30 t1t=a.cut(t1, k);
31 t2t=a.cut(t2, k);
32
33 }catch(MyException e){
34 return -1;
35 }
36 int he=a.heji(t1t, t2t);
37 int jiao=a.jiaoji(t1t, t2t);
38 return (float)jiao/he;
39 }
40
41 }
面板设计部分：
1 package bigproject2;
2 import java.awt.*;
3 import java.awt.event.*;
4 import java.io.BufferedReader;
5 import java.io.File;
6 import java.io.FileNotFoundException;
7 import java.io.FileReader;
8 import java.io.IOException;
9 import java.io.InputStreamReader;
10
11 import javax.swing.*;
12 import javax.swing.event.*;
13 import javax.swing.filechooser.FileNameExtensionFilter;
14
15 public class Outlook extends JFrame{
16 JFrame frm=new JFrame("相似度计算器");
17 JPanel areabottom=new JPanel();
18 JPanel areatop=new JPanel();
19 JPanel areamiddle=new JPanel();
20 static JTextArea tl=new JTextArea();
21 static JTextArea tr=new JTextArea();
22 JScrollPane left=new JScrollPane(tl,JScrollPane.VERTICAL_SCROLLBAR_ALWAYS,
23 JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);
24 JScrollPane right=new JScrollPane(tr,JScrollPane.VERTICAL_SCROLLBAR_ALWAYS,
25 JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);
26 JSplitPane sp=new JSplitPane(JSplitPane.HORIZONTAL_SPLIT,left,right);
27 static JButton toBig=new JButton("全部大写");
28 static JButton delbd=new JButton("去掉标点");
29 static JButton count=new JButton("计算相似度");
30 JLabel space=new JLabel(" ");
31 JLabel t1=new JLabel("Text1");
32 JLabel t2=new JLabel("Text2");
33
34 JMenuBar mb=new JMenuBar();
35 JMenu open=new JMenu("打开");
36 JMenuItem opent1=new JMenuItem("打开到Text1");
37 JMenuItem opent2=new JMenuItem("打开到Text2");
38
39 private String str="";
40 public Outlook()
41 {
42 judge();
43
44 frm.setVisible(true);
45 frm.setBounds(50, 50, 500, 400);
46 frm.setLayout(new BorderLayout(5,5));
47
48 frm.add("North",areatop);
49 frm.add("Center",areamiddle);
50 frm.add("South",areabottom);
51
52 areatop.add(mb);
53 mb.add(open);
54 open.add(opent1);
55 open.add(opent2);
56 open.setPreferredSize(new Dimension(40,18));
57 mb.setBackground(frm.getBackground());
58 areatop.setLayout(new FlowLayout(FlowLayout.LEFT));
59 areamiddle.setLayout(new FlowLayout(FlowLayout.LEFT));
60
61 areamiddle.add(t1);
62 t1.setPreferredSize(new Dimension(frm.getWidth()/2-20,10));
63 areamiddle.add(t2);
64 t2.setPreferredSize(new Dimension(50,10));
65 areamiddle.add(left);
66 left.setPreferredSize(new Dimension(frm.getWidth()/2-20,frm.getHeight()/2));
67 areamiddle.add(right);
68 right.setPreferredSize(new Dimension(frm.getWidth()/2-20,frm.getHeight()/2));
69 tl.setLineWrap(true);
70 tr.setLineWrap(true);
71
72 areabottom.add(toBig);
73 areabottom.add(delbd);
74 areabottom.add(space);
75 areabottom.add(count);
76
77 opent1.addActionListener(new ActionListener(){
78 public void actionPerformed(ActionEvent e) {
79 try {
80 openfile();
81 tl.setText(str);
82 } catch (IOException e1) {
83 e1.printStackTrace();
84 }
85 judge();
86 }
87 });
88 opent2.addActionListener(new ActionListener(){
89 public void actionPerformed(ActionEvent e) {
90 try {
91 openfile();
92 tr.setText(str);
93 } catch (IOException e1) {
94 e1.printStackTrace();
95 }
96 judge();
97 }
98 });
99 toBig.addActionListener(new ActionListener(){
100 public void actionPerformed(ActionEvent e){
101 tl.setText(tobig(tl.getText()));
102 tr.setText(tobig(tr.getText()));
103 }
104 });
105
106 delbd.addActionListener(new ActionListener(){
107 public void actionPerformed(ActionEvent e){
108 tl.setText(del(tl.getText()));
109 tr.setText(del(tr.getText()));
110 judge();
111 }
112
113 });
114 count.addActionListener(new ActionListener(){
115 public void actionPerformed(ActionEvent e){
116 KShinglingAlgorithm a=new KShinglingAlgorithm();
117 a.setText1(tl.getText());
118 a.setText2(tr.getText());
119 float b=a.getSimilarity(4);
120 if(b!=-1)
121 JOptionPane.showMessageDialog(null, Float.toString(b),"相似度", JOptionPane.INFORMATION_MESSAGE);
122 }
123 });
124 tr.addKeyListener(new KeyAdapter(){
125 public void keyTyped(KeyEvent e){
126 judge();
127 }
128 });
129 tl.addKeyListener(new KeyAdapter(){
130 public void keyTyped(KeyEvent e){
131 judge();
132 }
133 });
134 }
135 public void judge(){
136 if(tl.getText().length()!=0||tr.getText().length()!=0) {
137 toBig.setEnabled(true);
138 delbd.setEnabled(true);
139 count.setEnabled(true);
140 }
141 else{
142 toBig.setEnabled(false);
143 delbd.setEnabled(false);
144 count.setEnabled(false);
145 }
146 }
147 public void openfile() throws IOException{
148 str="";
149 JFileChooser choose=new JFileChooser();
150 int result = choose.showOpenDialog(this);
151 File file = null; //注意初始化
152 //加过滤器
153 if (result == JFileChooser.APPROVE_OPTION) {
154 file = choose.getSelectedFile();
155 }
156 else{
157 return; //使点取消后不会抛出异常
158 }
159 FileReader fr=new FileReader(file);
160 BufferedReader br=new BufferedReader(fr);
161 char c[]=new char[512];
162 String strline="";
163 while(br.ready()){
164 strline=br.readLine();
165 str+=strline;
166 };
167 br.close();
168 fr.close();
169 }
170 public String tobig(String str){
171 String temp="";
172 for(int i=0;i<str.length();i++)
173 {
174 if(str.charAt(i)>=‘a‘&&str.charAt(i)<=‘z‘)
175 {
176 char t=str.charAt(i);
177 t=(char)(str.charAt(i)-32);
178 temp+=t;
179 }
180 else temp+=str.charAt(i);
181 }
182 return temp;
183 }
184
185 public String del(String str){
186 String temp="";
187 for(int i=0;i<str.length();i++)
188 {
189 char t=str.charAt(i);
190 if(t>=‘!‘&&t<=‘/‘||t>=58&&t<=64||t>=91&&t<=96||t>=123&&t<=126);
191 else temp+=t;
192 }
193 return temp;
194 }
195 public static void main(String[] args){
196 new Outlook();
197
198
199 }
200 }
Outlook
时间： 2024-11-08 17:32:41