Improve duplicate handling

getter.py

@@ -70,16 +70,18 @@ while True:
      print "Error in " + str(twid)
      print e
     continue
-
-  words = text.split(" ")
-  for word in words:
-   if word[0] == "@" and (not user == word[1:]) and (not sender == word[1:]):
-    wordlist += word
-
-  wordlist = list(set(wordlist))
-  for word in wordlist:
-   comment += word + " "
-
+  
+  words = text.split(" ")  
+  for word in words:  
+   wordlist += word  
+  
+  wordlist = list(set(wordlist))  
+  try:  
+   wordlist.remove(user)  
+   wordlist.remove(sender)  
+  except ValueError:  
+   pass  
+  
   cur.execute("INSERT INTO tweets VALUES(%i,'%s','%s','%s',0)" % (twid,date.strftime("%Y-%m-%dT%H:%M:%S"),sender,comment.strip()))
   sql_conn.commit()