#! /usr/bin/python
import re
import urllib
from BeautifulSoup import BeautifulSoup
from elementtree.SimpleXMLWriter import XMLWriter
import logging
import time
# -*- coding: utf-8 -*-

def remove_values_from_list(the_list, val):

	return [value for value in the_list if value != val]

def formatage(text):
	
	texte = (" ".join(text.split()))
	return texte.replace(" ","")

def formatagetitre(table,myurl):
	
	titre = ""
	try:
		titre = (table.find("h1").renderContents().replace('\n','')).decode("utf-8")
	except:
		logging.info("Erreur inclusion Titre %s: %s" % ( titre,myurl))
	return titre

def formatagemap(table,myurl):
	
	date = ""
	try:
		date = table.find("span",{"class" : "date"}).renderContents().split(",")[1].replace("le ","").lstrip().replace(" ","-")
	except:
		logging.info("Erreur inclusion Date %s: %s" % ( date,myurl))
	return date

def formatagelieu(table,myurl):
	
	lieu = ""
	try:
		lieu = table.find("span",{"class" : "bold"}).find("a").string
	except:
		logging.info("Erreur inclusion Lieu %s: %s" % ( lieu,myurl))
	return lieu

def formatagelist(table,myurl):
		
	niveau = ""
	try:
		temp = table.findAll("strong")
		liste = (re.sub("au n","n",temp[1].renderContents()))
	except:
		try:
			element = "niveau"
			index = str(table).find(element)
			niveau = str(table)[index:index + 8]
			logging.info("Erreur inclusion Niveau Autre %s: %s" % ( niveau,myurl))
		except:
			logging.info("Erreur inclusion Niveau %s: %s" % ( niveau,myurl))

	return niveau

def formatagecontent(table,myurl):
	
	content = ""
	try:
		divp = table.find("p",{"class":"pdf_link"})
		divp.extract()
		divblock1 = table.find("div",{"class":"block1"})
		divblock1.extract()
		for div in table.findAll("span"):
			div.extract()
		divblockul = table.find("ul",{"class":"list"})
		divblockul.extract()
		content = formatage("".join(table.findAll(text=True)))
	except:
		logging.info("Erreur inclusion Content %s: %s" % ( content,myurl))
	return content

def rassemblement():

	listurl= []
	''' 61 au lieu de 1'''
	for i in range(0,66):
		if i<1: 
			url = "http://www.asn.fr/index.php/content/view/full/1524//%28offset%29/"
		else:
			url = "http://www.asn.fr/index.php/content/view/full/1524//%28offset%29/"+str(int(i*20))
		f = urllib.urlopen(url)
		htmldata = f.read()
		soup = BeautifulSoup(htmldata,convertEntities=BeautifulSoup.HTML_ENTITIES)
		table3 = soup.find("div", { "id" : "min_height" })
		table = table3.findAll( 'a',href = re.compile("^/index.php/Les-activites-controlees-par-l-ASN/Production-d-Electricite/Avis-d-incidents-dans-les-installations-nucleaires/.*"))
		for element in table:
			listurl.append(element['href'])
		table2 = table3.findAll( 'a',href = re.compile("^/index.php/content/view/full/.*"))
		for element in table2:
			match = re.match("^/index.php/content/view/full/1524/.*",element['href'])	
			if match is not None:
				pass
			else:
				match = re.match("^/index.php/content/view/full/900/.*",element['href'])	
				if match is not None:
					pass
				else:
					listurl.append(element['href'])
		i = i + 1
		f.close()
	return listurl

def traitement(myurl):
	
	w.start("incident")
	w.element("URL","http://www.asn.fr/" + myurl)
	f = urllib.urlopen("http://www.asn.fr/" + myurl)
	htmldata = f.read()
	
	soup = BeautifulSoup(htmldata)
	table = soup.find("div",{"class" : "col4 entry"})
	w.element("titre",formatagetitre(table,myurl))

	w.element("date",formatagemap(table,myurl))

	w.element("lieu",formatagelieu(table,myurl))
	w.element("niveau",formatagelist(table,myurl))

	table2 = soup.find("div",{"id" : "min_height"})
	w.element("text",formatagecontent(table2,myurl))
	w.end("incident")

if __name__ == '__main__': 

	logging.basicConfig(filename='erreur.log',level=logging.INFO,format='%(asctime)s %(levelname)s - %(message)s',datefmt='%d/%m/%Y %H:%M:%S')
	listurl = rassemblement()
	listurl = remove_values_from_list(listurl,"/index.php/content/view/full/1524")
	w = XMLWriter("asn.xml","utf-8")
	w.declaration()
	
	for element in listurl:
		time.sleep(10)		
		print "http://www.asn.fr/" + element
		traitement(element)
	
	playlist = w.start("asn")
	w.close(playlist)