CSC352 Homework 2 Solution 1

--D. Thiebaut 22:38, 22 February 2010 (UTC)

---

#! /usr/bin/python
# D. Thiebaut
# threadedRetrieveEtext.py
#
# This program gets a keyword from the command line, and 
# launches a thread that accesses a server and prompts it for
# a list of 20 Urls of files containing that keyword.
# The program then launches 20 threads on the 20 different threads
# and parallelizes the wait of the 20 threads.
# When the 20 threads are done, the main program gets the 
# excerpts from each file with a few words before and after the 
# keyword, and prints the results on the screen.

import sys
import urllib, urllib2
import textwrap
import threading
import Queue        # use "queue" lowercase in Python 3
import time
import random

MAINURL = "http://xgridmac.dyndns.org/~thiebaut/swish-e/swishe.php"
VERSION = """Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11)
 Gecko/20071127 Firefox/2.0.0.11"""
HEADERS       = {"User-Agent"  : VERSION }



class urlRetrieverThread( threading.Thread ):
    
    def __init__( self, url, keyword, queue=None, Id=None ):
        threading.Thread.__init__( self )
        self.Id        = Id
        self.queue     = queue
        self.url       = url      # the url of the text/page to be retrieved
        self.text      = None     # the page retrieved
        self.keyword   = keyword  # the keyword to search for, if any
        self.args      = {}
        if keyword is not None:
            self.args = { "search" : keyword }
            
    def getPage( self ):
        """returns the page retrieved by this thread.  Make sure
        isAlive() returns false first, to make sure the thread has
        finished before calling this."""
        return self.text

    def getExerpt( self ):
        """returns the 1st portion of the page retrieved by this thread
        surrounding the current keyword.  Make sure
        isAlive() returns false first, to make sure the thread has
        finished before calling this."""
        if self.keyword is None or self.text is None:
            return ""
        
        offset = self.text.find( self.keyword ) 
        exerpt = self.text
        if offset == -1:
            return ""
        exerpt = self.text[ max(offset-60, 0)
                          : min( offset+60, len(self.text) )]
        
        #--- remove duplicate white spaces and wrap at 40 chars---
        exerpt = ' '.join( exerpt.split()[1:-1] )
        exerpt = '\n'.join( textwrap.wrap( exerpt, 40 ) )
        return exerpt

    def run( self ):        
        req = urllib2.Request( self.url,
                               urllib.urlencode( self.args ),
                               HEADERS )
        f = urllib2.urlopen( req )
        self.text = f.read()
        
        #--- Just to see what is happening, this code section
        #--- will add some additional delay to see how result information                                              
        #--- from all the threads evolved with time.                                                               
        #--- Removee this line for normal operation!
        time.sleep( random.randint( 1, 50 ) /10 )  

        #--- tell main program we're done ---
        if self.queue is not None:
            self.queue.put( self.Id )
            
def parse( text, debug=False ):
    lines = text.split( "\n" )
    list = []
    doc  = {}
    for line in lines:
        if line.find( "<br>url:" )!=-1: 
            doc["url"] = line.split( "url:" )[1].strip() 
        if line.find( "<br>score:" )!=-1: 
            doc["score"] = int( line.split( "score:" )[1].strip() ) 
        if line.find( "<br>file:" )!=-1: 
            doc["file"] = line.split( "file:" )[1].strip() 
        if line.find( "<br>offset:" )!=-1: 
            doc["offset"] = int( line.split( "offset:" )[1].strip() ) 
            list.append( doc )
            doc = {}
        
    return list


def getAllDocs( listOfDocs, keywords, debug=False ):
    for doc in listOfDocs:
        url = doc[ "url" ]
        fileName = doc[ "file" ]
        if debug:
            print "--------------------------------------------"
            print "fetching url:", url
        text = getOneDoc( url, keywords, debug )
        if debug: print text        


def goHome():
    """bring cursor to top left position on the screen"""
    sys.stderr.write( "\x1b[0;0H\x1b[2J" )

def displayResults( results ):
    """ update the list of results on the screen"""
    results.sort( )
    results.reverse( )
    goHome()
    for i in range( len( results ) ):
        score, url, exerpt = results[ i ]
        print ""
        print "%d %s" % (score, url)
        print " -- ", ' '.join( exerpt.split( "\n" ) )

  
def main( debug=False ):
    if len( sys.argv ) > 1:
       keyword = sys.argv[1]

    if debug: print "starting main thread"
    mainThread = urlRetrieverThread( MAINURL, keyword )
    mainThread.start()
    mainThread.join()

    if debug: print "main thread done!"
    
    text = mainThread.getPage()
    if debug: print "list of docs = ",text

    listOfDocs = parse( text, debug )

    #--- create a queue for communication between threads ---
    #--- and main                                          ---
    queue = Queue.Queue( 0 )
        
    #--- start all the threads ---
    listOfThreads = {}
    for id, doc in enumerate( listOfDocs ):
        url = doc[ "url" ]
        thread = urlRetrieverThread( url, keyword, queue, id )
        listOfThreads[id] = (doc, thread)
        thread.start()
        if debug: print "starting thread", id
        

    #--- wait for the threads to pass their Id back through the queue---
    results = []
    count = 0
    while True:
        #--- get the Id of the thread that is done ---
        id = queue.get( block=True, timeout=None )

        #--- get the info and the thread pointer associated with this id ---
        doc, thread = listOfThreads[id]
        score  = doc[ "score" ]
        url    = doc[ "url" ]
        if debug:
            print "dequeued", id, score, url

        #--- get the bit of text around the keyword from the thread ---
        exerpt = thread.getExerpt()
        if debug:
            print "exerpt = ", exerpt
            
        #--- add a triplet to the list ---
        results.append( [score, url, exerpt] )

        #--- update the display with results sorted by scores ---
        displayResults( results )
        
        #--- stop when we have received all the texts back ---
        count += 1
        if count >= len( listOfDocs ):
            break
        
    if debug:
        print "main done!"

        
main( False )