import urllib2
import time

# load words from the file 
def load_file(file_name):
    file = open(file_name, 'r')
    list_tv = []

    while 1:
        line = file.readline()
        if not line:
          break
        pass
        word = line.strip()
        if len(word) > 0:
            list_tv.append(word.lower())

    return list_tv
   

# save the words of a list
def save_file(list_words, file_name):
    file = open(file_name, 'w')

#   uniques = set(list_words)
#   for w in uniques:
    for w in list_words:
       file.write(w + '\n')


# append the internal language found in the file
def append_to_file_ITL(internallang):
    with open("internallang.txt", "a") as myfile:
        for itl in internallang:
            myfile.write(itl + "\n")

# save the HTML in file
def save_content_file(content):
    file = open("html.txt", 'w')
    file.write(content)



# read the HTML content of a URL
def read_url(word):
    return urllib2.urlopen("http://thesaurus.com/browse/" + word).read()



# get the data from a HTML tag
from HTMLParser import HTMLParser

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data().strip()


# valid word only have lowercase letters
import re
def is_valid(word, search=re.compile(r'[^a-z.]').search):
    return not bool(search(word))









# extract the language data from the HTML
def parseHTML(content):
    intLang = []
    synos = []
    ini = 20000  # 34000
    
    while 1:
        pos1 = content.find("Main Entry:", ini);
        if pos1 == -1: 
           break

        pos2 = content.find("Part of Speech:", pos1);
        if pos2 == -1: 
           break

        word = strip_tags(content[pos1 + len("Main Entry:"):pos2]).split('\n')[0]

        if len(word.split(' ')) > 1:
            ini = pos2
            continue

        pos3 = content.find("Definition:", pos2);
        if pos3 == -1: 
           break

        ini = pos3
        
        tipe = strip_tags(content[pos2 + len("Part of Speech:"):pos3]).split('\n')[0]  

        if tipe == 'verb' or tipe == 'adjective' or tipe == 'adverb':
            codtype = "J"
            if tipe == 'verb':
                codtype = "V"
            if tipe == 'adverb':
                codtype = "B"
            
            intLang.append("WORD " + codtype + " " + word + " " + word)
            continue

        if tipe != 'noun':
            continue

        pos4 = content.find("Synonyms:", pos3);
        if pos4 == -1: 
           break

        defi = strip_tags(content[pos3 + len("Definition:"):pos4])

        words = word.split("/")
        for wd in words:
           intLang.append(wd + " is a " + defi)

        pos5 = content.find("</table>", pos4);
        if pos5 == -1: 
           break
		        	
        pos6 = content.find("Antonyms:", pos4);
        if pos6 > 0 and pos6<pos5:
            pos5 = pos6;
        
        syno = strip_tags(content[pos4 + len("Synonyms:"):pos5]).split(',')

        max = 0
        for s in syno:
            w = s.strip()
            if max < 2 and is_valid(w):
                max = max + 1
                synos.append(w)


        ini = pos5
        
        
    # here the end of the loop
    return intLang, synos

# here the end of the proc    










#main 

to_visit_list = load_file("tovisit.txt")
visited_list = load_file("visited.txt")


times = 10
i = 0
while i < times:
    if len(to_visit_list) == 0:
        break
    
    tovisit=to_visit_list.pop(0)

    if tovisit not in visited_list:
        content = read_url(tovisit)

        save_content_file(content)

        internallang, synonims = parseHTML(content)

        append_to_file_ITL(internallang)

        for syn in synonims:
            to_visit_list.append(syn)            
        
        print tovisit + " " + str(len(internallang)) + " / " + str(len(synonims))
        visited_list.append(tovisit)

        i = i + 1
        time.sleep(2)

    else:
        print tovisit + " visited!!"

    
to_visit_list.sort()
to_visit_list.reverse()

save_file(to_visit_list, "tovisit.txt")
save_file(set(visited_list), "visited.txt")