import logging import re from sets import Set from datetime import datetime from difflib import SequenceMatcher from django.db import connection from django.http import HttpResponse from django.template import RequestContext, loader from django.conf import settings def htmlspecialchars(text): """Inlocuim caracterele problema din codul HTML. """ chars = ( ('\xe2\x80\x93', '-'), ('\xe2\x80\x9e', '"'), (r'\xc4\x83', 'a'), (r'\xc8\x99', 's'), (r'\xc3\xae', 'i'), (r'\xc8\x9bi', 't'), (r'\xc3\xa2', 'a'), (r'\xc5\xa3', 't'), (r'\xc5\x9f', 's'), (r'\xc3\x8e', 'I'), (r'\r', ' '), (r'\n', ' '), (r'\xc8\x9b', 't'), (r'\xc8\x98', 'S'), (r'\t', ' '), (r'\xc4\x82', 'A'), (r'\xc5\xa2', 'T'), (r'\xe2\x80\x93', '-'), (r'\xe2\x80\x99', "'"), (r'\xe2\x80\x9e', '"'), (r'\xe2\x80\x9c', '"'), (r'\xc5\x9e', 'S'), (r'\xc2\xab', ''), (r'\xc2\xbb', ''), (r'\xc3\xa9', 'e'), ('\xc3\x8e', 'I'), ('\u0102\u0179', ''), ('\xc4\x83', 'a'), ('\xc5\xa3', 't'), ('\xc5\x9f', 's'), ('\xc5\xa2', 'T'), ('\xce', 'I'), ('\x96', '-'), ('„', '"'), ('…', '...'), ('ţ', 't'), ('°', ' gr. '), ('\xaa', 'S'), ('Ţ', 'T'), ('Ă', 'A'), ('Â', 'A'), ('â', 'a'), ('&', '&'), ('\xfe', 't'), ('\xba', 's'), ('Ş', 'S'), ('\xe3', 'a'), ('\xee', 'i'), ('\xe2', 'a'), ('î', 'i'), ('ã', 'a'), ('\xc3\xae', 'i'), ('\x85', '...'), ('\xe2\x80\x9c', '"'), ('\xc3\xa2', 'a'), ('\xc3\xa7', 'I'), ('Î', 'I'), ('ă', 'a'), ('ş', 's'), ('â', 'a'), ('”', '"'), ('“', '"'), ('\xc7', '<<'), ('\xc8', '>>'), ('”', '"'), ('•', ' '), ('î', 'i'), ('â', 'a'), ('ş', 's'), (' ', ' '), ('“', '"'), ('–', '-'), ('\n', ' '), ('\r', ' '), ('’', "'"), ('\t', ' '), (''', "'"), ('"', '"'), ('\x94', '"'), ('\x84', '"'), ('\x93', '"'), ('\x92', "'"), ('\x95', " "), ('a\x80\x9c', '"'), ('a\x80\x9d', '"'), ('\xc5\x9e', 'S'), ('\xde', 'T'), ('\xc3', 'A'), ('\xc2\x8c', 'I'),('\xc2', 'A'), ('\xd6', 'O'), ('„', '"'), ('ş', 's'), ('“', '"'), ('«', '"'), ('ă', 'a'), ('»', '"'), ('”', '"'), ('ţ', 't'), ('Î', 'I'), ('>', '-'), ('º', 's'), ('þ', 't'), ('ã', 'a'), ('Ş', 'S'), ('ă', 'a'), ('ş', 's'), ('â', 'a'), ('ţ', 't'), ('î', 'i'), ('Î', 'I'), ('–', '&'), ('\xe1', 'a'), ('\xc1', 'A'), ('•', ''), ('\xe9', 'e'), ('\xe8', 'e'), ('\xc9', 'E'), ('&ndash', '&'), ('ü', 'u'), ('‘', "'"), ('é', 'e'), ('Ş', 'S'), ('Ţ', 'T'), ('\xB6lA\xB6', 'a'), ('\x80\x99', "'"), ('<', ''), ('>', ''), ('\xf6', 'o'), ('\xf5', 'o'), ('\xf3', 'o'), ('\xd3', 'O'), ('\xd5', 'O'), ('\xdc', 'U'), ('\xe7', 'c'), ('\xed', 'i'), ('\xfc', 'u'), ('\xe4', 'a'), ('­', ''), ('ă', 'a'), ('ş', 's'), ('ş', 's'), ('â', 'a'), ('ţ', 't'), ('î', 'i'), ('Î', 'I'), (r'\xe2\x80\xa6', '...'), (r'\xe2\x80\x9d', '"'), (r'\xe2\x80\x9d', '"'), (r'\xc2\xa0\xc2\xa0', ''), (r'\xc2\xa0', ''), (r'\xc3\xa3', 'a'), (r'\xc2\xaa', 'S'), (r'\xc3\x82', 'A'), (r'\xc2\xad', ''), (r'\xe2\x97\x8f', ''), ('-->', ''), ('ó', 'o'), ('í', 'i'), ('ö', 'o'), (r'\xc3\xa0', 'a'), (r'\xc8\x9a', 'T'), (r'\xc3\xbc', 'u'), (r'\xc3\xa1', 'a', ), (r'\xe2\x80\x98', '"'), (r"\'\'", ''), (r"\'", ''), (r'\x96', '-'), (r'\xe2', 'a'), (r'\x92', "'", ), (r' ', '', ), (r'\x95', '-'), (r'\x84', '"'), (r'\xee', 'i'), (r'a\x80\xa2', ''), (r'\xe3', 'a'), (r'\xce', 'I'), (r'\xc3\xb6', 'o'), (r'\xc3\xab', 'e'), (r'\xc3\xbb', 'u'), (r'\xc3\xa4', 'a'), (r'\xc3\xad', 'i'), (r'\xc3\xb3', 'o'), (r'\xc3\xa7', 'c'), ('’', "'"), ('…', '...'), ) for item in chars: try: text = text.replace(item[0], item[1]) except UnicodeDecodeError: pass return text def get_news_logger(): """Logger pentru aplicatie. """ logger = None if settings.NEWS_DEBUG: logger = logging.getLogger('my_logger') logger.setLevel(logging.DEBUG) logger.addHandler(logging.FileHandler(settings.NEWS_DEBUG_FILE)) return logger def profile_sql(last_queries=None, compute_duplicates=False, print_to_output=False): queries = connection.queries counter = 1 prag_duplicate = 2 nr_queries = len(queries) if print_to_output: print 'Numar de query-uri sql = %d' % (len(queries)) else: logging.debug('Numar de query-uri sql = %d' % (len(queries))) if compute_duplicates: dict_duplicates = {} for query in queries: if dict_duplicates.has_key(query['sql']): dict_duplicates[query['sql']] += 1 else: dict_duplicates[query['sql']] = 1 temp_list = [[value, sql] for sql, value in dict_duplicates.iteritems() if value >= prag_duplicate] temp_list.sort() total_duplicate = sum([item[0] for item in temp_list]) if nr_queries: procent_duplicate = ((total_duplicate * 1.0) / nr_queries) * 100 else: procent_duplicate = 0 if print_to_output: print 'Avem un total de %d duplicate, dintr-un total de %d queries, i.e. %f%% din total.' % (total_duplicate, nr_queries, round(procent_duplicate, 2)) else: logging.debug('Avem un total de %d duplicate, dintr-un total de %d queries, i.e. %f%% din total.' % (total_duplicate, nr_queries, round(procent_duplicate, 2))) if last_queries and queries: queries = queries[(-1) * last_queries:] for query in queries: if print_to_output: print '%d) query: %s , time = %s' % (counter, query['sql'], str(query['time'])) else: logging.debug('%d) query: %s , time = %s' % (counter, query['sql'], str(query['time']))) counter += 1 def custom_render_to_response(request, template, dict_return): """Custom rendere. """ t = loader.get_template(template) c = RequestContext(request, dict_return) response = HttpResponse(t.render(c)) return response def encode_url(url): """Incodarea unui URL. """ url = url.replace(' ', '%20') return url def decode_url(url): """Decodarea unui URL. """ url = url.replace('%20', ' ') return url def elimina_javascript(text): """Functie pentru a elimina JavaScript dintr-un anumit text. """ start_pat = '<[ \n\r]*script[^>]*>' end_pat = '<[ \n\r]*/script[^>]*>' s1 = re.search(start_pat, text) s2 = re.search(end_pat, text) if s1 and s2: text = text[:s1.start()] + text[s2.start()+len(s2.group()):] return text def clean_text(text): """Sterge textul de caracterele html si de spatiile si end-line-urile nedorite """ text = clean_out_ie_html_comments(text) text = htmlspecialchars(text) text = re.sub('\n', ' ', text) text = re.sub('\r', ' ', text) #let's assume we have at most 3 javascript thingies in the code text = elimina_javascript(text) text = elimina_javascript(text) text = elimina_javascript(text) text = elimina_javascript(text) text = re.sub('<.*?>', ' ', text) text = re.sub('', '', text) #text = re.sub('', '', text) text = re.sub(' {2,}', ' ', text) text = text.strip() return text def clean_out_ie_html_comments(text): """"Observator cultural" has some strange HTML code comments that I need to clean out """ inc = 0 temp_list = [0] while inc != -1: inc = text.find('', inc) if end != -1: end += len('') temp_list.append(inc) temp_list.append(end) temp_list.append(len(text)) temp_list2 = [] if len(temp_list) > 2 and (not len(temp_list)%2): for j, item in enumerate(temp_list): if not j%2: temp_list2.append(text[temp_list[j]:temp_list[j+1]]) if temp_list2: text = ' '.join(temp_list2) return text def safe_unicode(obj, *args): """ return the unicode representation of obj vezi http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466341""" try: return unicode(obj, *args) except UnicodeDecodeError: # obj is byte string ascii_text = str(obj).encode('string_escape') return unicode(ascii_text) def safe_str(obj): """ return the byte string representation of obj vezi http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466341""" try: return str(obj) except UnicodeEncodeError: # obj is unicode return unicode(obj).encode('unicode_escape') def clean_and_safe(text): return clean_text(safe_str(text)) def string_distance(text1, text2): """Intoarce distanta dintre text1 si text2 """ s = SequenceMatcher(lambda x: x == " ", text1, text2) return s.ratio() def list_diff(list1, list2): """Intoarce datele care se afla in list1 si nu se afla in list2 """ diff_set = Set(list1) - Set(list2) return [item for item in diff_set] def scade_luna(timp, diff): #scade diff luni din timp res = timp.month - diff carry = abs(res/12) if res%12 == 0: carry +=1 month = res%12 if month == 0: month = 12 year = timp.year - carry return datetime(year, month, 1)