Input:
Dissy,Alice Bob Cissy Emma
Bob,Alice Cissy Dissy Emma
Alice,Bob Cissy Dissy
Emma,Bob Cissy Dissy
Cissy,Alice Bob Dissy Emma
Download/copy jar datafu-0.0.5.jar to HDFS location
Go To Pig Grunt
REGISTER <HDFS_PATH>/datafu-0.0.5.jar ;
define SetIntersect datafu.pig.bags.sets.SetIntersect();
A = LOAD '/user/sansari2/poc/file1' USING PigStorage(',') AS
(key:chararray,val1:chararray);
B = FOREACH A GENERATE key, FLATTEN(TOKENIZE(val1)) as token
; -- table 1
C = FILTER B BY (key < token);
D = FOREACH A GENERATE key, TOTUPLE(TOKENIZE(val1)) as token
;
E = FOREACH D GENERATE $0 AS key:chararray,
FLATTEN(TOBAG($1..)) as val1; -- table 2
F = JOIN B by key FULL OUTER, E by key;
G = JOIN F by B::token FULL OUTER, E by key;
H = FOREACH G GENERATE E::key as friend1, F::B::key as
friend2, E::val1 as group1, F::E::val1 as group2;
I = FOREACH H {
s1 = ORDER group1 by token;
s2 = ORDER group2 by token;
GENERATE friend1, friend2, SetIntersect(s1,s2);
};
J = FILTER I by (friend1 < friend2);
K = ORDER J BY friend1,friend2 ASC;
Output:
(Alice,Bob,{(Cissy),(Dissy)})
(Alice,Cissy,{(Bob),(Dissy)})
(Alice,Dissy,{(Bob),(Cissy)})
(Bob,Cissy,{(Alice),(Dissy),(Emma)})
(Bob,Dissy,{(Alice),(Cissy),(Emma)})
(Bob,Emma,{(Cissy),(Dissy)})
(Cissy,Dissy,{(Alice),(Bob),(Emma)})
(Cissy,Emma,{(Bob),(Dissy)})
(Dissy,Emma,{(Bob),(Cissy)})
