# HG changeset patch
# User Ivo Hinkelmann <ihi@openoffice.org>
# Date 1295018266 -3600
# Node ID 9cf541f20652b9218d4b93449607b5ba72aa7abb
# Parent  4fc2c733c3677e5b8908d20956c30d5e884291da
#i116350# Change sdf file format

diff --git a/l10ntools/scripts/tool/init_sdf2.py b/l10ntools/scripts/tool/init_sdf2.py
--- a/l10ntools/scripts/tool/init_sdf2.py
+++ b/l10ntools/scripts/tool/init_sdf2.py
@@ -47,7 +47,7 @@
         sdf2_data = sdf2.convert_sdf_to_sdf2Data(sdf_source_lines, sdf_data_lines)
         try:
             fh = open(options.data, "w")
-            for line in sdf2_data:
+            for line in sorted(sdf2_data, key = lambda a: str(a)):
                 fh.write(str(line)+"\n")
         except IOError:
             print "ERROR: Can not write file " + options.data
@@ -58,7 +58,7 @@
         sdf2_ident= sdf2.convert_sdf_to_sdf2Ident(sdf_source_lines)
         try:
             fh = open(options.ident, "w")
-            for line in sdf2_ident:
+            for line in sorted(sdf2_ident, key = lambda a: str(a)):
                 fh.write(str(line)+"\n")
         except IOError:
             print "ERROR: Can not write file " + options.ident
diff --git a/l10ntools/scripts/tool/l10n.py b/l10ntools/scripts/tool/l10n.py
--- a/l10ntools/scripts/tool/l10n.py
+++ b/l10ntools/scripts/tool/l10n.py
@@ -1,4 +1,4 @@
-import sys, l10nupdate, l10nverify, l10nstatus
+import sys, l10nupdate, l10nverify, l10nstatus, l10nimport
 
 def usage():
     print "help!"
@@ -11,6 +11,8 @@
 
 if(cmd == "update"):
     l10nupdate.run()
+if(cmd == "cleanup"):
+    l10nupdate.cleanup()
 elif(cmd == "verify"):
     l10nverify.run()
 elif(cmd == "status"):
diff --git a/l10ntools/scripts/tool/l10nimport.py b/l10ntools/scripts/tool/l10nimport.py
--- a/l10ntools/scripts/tool/l10nimport.py
+++ b/l10ntools/scripts/tool/l10nimport.py
@@ -1,72 +1,78 @@
-import sdf, sdf2, sys, tempfile
+import sdf, sdf2, sys, tempfile, os
 
-def run(opts)
+def run(opts):
     x = l10nimport(opts)
     x.do_import()
 
 class l10nimport():
-    def __init__(opts):
+    def __init__(self, opts):
         self.opts = opts
         self.filename = ""
 
     def do_import(self):
-        if(len(opts) == 2):
-            self.filename = opts[1]
+        if(len(self.opts) == 3):
+            self.filename = self.opts[2]
         else:
             print "Usage: l10n import <file>"
             sys.exit(13)
 
-        lines = self.read_file(filename)
+        lines = self.read_file(self.filename)
         new_dict = dict()
-        data = sdf2.convert_sdf_to_sdf2Data("", lines)
-        for new_line in data:
-            new_dict[new_line.keyid] = new_line
+        for line in lines:
+            sdfl = sdf.SdfEntity()
+            sdfl.set_properties(line)
+            data = sdf2.convert_sdfentity_to_sdf2data(sdfl)
+            new_dict[data.keyid] = data
 
         fh = None
-		cur_lang = ""
-		localize_dict = None
-		
+        cur_lang = ""
+        localize_dict = None
+        
         for keyid,de in new_dict.iteritems():
-			if de.langid == "en-US":
-				pass
-			elif not de.langid == cur_lang:
-				if localize_dict != None:
-					self.write_sdf2_data(cur_lang, localize_dict)
-				cur_lang = de.langid	  	
-				localize_dict = self.read_sdf2_data(de.langid)
-				localize_dict[de.keyid] = de
-			elif de.langid == cur_lang		
-				localize_dict[de.keyid] = de
-			else:
-				print "Not reachable, remove me!"
+            if de.lang == "en-US":
+                pass
+            elif not de.lang == cur_lang:
+                if localize_dict != None:
+                    self.write_sdf2_data(cur_lang, localize_dict)
+                cur_lang = de.lang          
+                localize_dict = self.read_sdf2_data(de.lang)
+                localize_dict[de.keyid] = de
+                #print "added " + str(de)
+            elif de.lang == cur_lang:        
+                localize_dict[de.keyid] = de
+                #print "added " + str(de)
+            else:
+                print "Not reachable, remove me!"
+        self.write_sdf2_data(cur_lang, localize_dict)
 
-	def write_sdf2_data(lang, localize_dict):
-		lines = sorted(localize_dict.values(), key=lambda a: str(a))
-		#todo: write to temp and shutil.move
-		try:
-			fh = open(os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"localize.sdf2", "w")
-			for line in lines:
-				fh.write(line+"\n")
-		except:
-			print "Can not write to file "+ os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"localize.sdf2"
-		fh.close()
+    def write_sdf2_data(self, lang, localize_dict):
+        lines = sorted(localize_dict.values(), key=lambda a: str(a))
+        #todo: write to temp and shutil.move
+        try:
+            fh = open(os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"/localize.sdf2", "w")
+            for line in lines:
+                fh.write(str(line)+"\n")
+        except IOError:
+            print "Can not write to file "+ os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"/localize.sdf2"
+        fh.close()
 
-	def read_sdf2_data(lang):
-    	localize_dict = dict()
-        sdf_data = SdfData(os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"localize.sdf2")
-        sdf_data.read()
+    def read_sdf2_data(self, lang):
+        localize_dict = dict()
+        sdf_data = self.read_file(os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/"+lang+"/localize.sdf2")
                 
-        for line in sdf_data.iteritems():
-        	se = SdfEntity()
+        for line in sdf_data:
+            se = sdf2.Sdf2DataEntity()
             se.set_properties(line)
-            localize_dict[line.keyid] = se
-		return localize_dict
+            localize_dict[se.keyid] = se
+        return localize_dict
 
-    def read_file(filename)
+    def read_file(self, filename):
         try:
             f = open(filename, "r")
-            lines = [lines.strip("\n") for line in f.readlines()]
-        except:
-            print "Can not read file "+file
+            lines = [line.strip("\n") for line in f.readlines()]
+        except IOError, e:
+            print "Can not read file "+filename
+            print e
             sys.exit(13)
+        f.close()
         return lines
diff --git a/l10ntools/scripts/tool/l10nstatus.py b/l10ntools/scripts/tool/l10nstatus.py
--- a/l10ntools/scripts/tool/l10nstatus.py
+++ b/l10ntools/scripts/tool/l10nstatus.py
@@ -98,7 +98,9 @@
 
         if cmd == "new" or cmd == "export":
             for x in sorted(status.new[lang].values(), key=lambda a: str(a)):
-                print str(sdf2.convert_sdf2idententity_to_sdf(self.ident[x.keyid]))
+                sdf = sdf2.convert_sdf2idententity_to_sdf(self.ident[x.keyid])
+                sdf.langid = lang
+                print str(sdf)
         
         if cmd == "changed" or cmd == "export":
             tmplist = []
diff --git a/l10ntools/scripts/tool/l10nupdate.py b/l10ntools/scripts/tool/l10nupdate.py
--- a/l10ntools/scripts/tool/l10nupdate.py
+++ b/l10ntools/scripts/tool/l10nupdate.py
@@ -1,11 +1,15 @@
-import tempfile, sys, subprocess, sdf_raw, sdf2, os, sdffile, heapq
+import tempfile, sys, subprocess, sdf_raw, sdf2, os, sdffile, heapq, glob
 
 # run localize
 # add all new strings to sourcedb
 
 def run():
     a = l10nupdate()
-    a.update_ident_file()
+    a.update_ident_file("update")
+
+def cleanup():
+    a = l10nupdate()
+    a.update_ident_file("cleanup")
 
 class l10nupdate:
     # fix me ;)
@@ -50,12 +54,13 @@
         #os.remove(tmpfile)
         return sdffile
     
-    def update_ident_file(self):
+    def update_ident_file(self, cmd):
         sdf_file = self.run_localize()
         ident = sdffile.IdentFile()
         ident.read(self.default_ident_location)
         unchanged_cnt, new_cnt, changed_cnt, moved_cnt, obsolete_cnt = 0, 0, 0, 0, 0
         new_dict, obsolete_dict = dict(), dict()
+        
         for line in sdf_file:
             status, c_ident = ident.add_sdf(line)
             if status == "UNCHANGED":
@@ -65,25 +70,44 @@
                 new_dict[c_ident.source_hash] = c_ident
             elif status == "CHANGED":
                 changed_cnt += 1
+        
+        cleanup_list = []
         for id,line in ident.ident.iteritems():
-            if line.get_l10n_status() == "":
+            if line.get_l10n_status() == "" and cmd == "update":
                 obsolete_dict[str(line)] = line
                 obsolete_cnt += 1 
+            elif line.get_l10n_status() == "" and cmd == "cleanup":
+                cleanup_list.append(line)
+        
+        if cmd == "cleanup":
+            print "Removeing obsolete strings from sdf2 ident file"
+            for line in cleanup_list:
+                del ident.ident[line.get_identifier()]
+            self.clean_sdf2_data(cleanup_list)
+            
         for key,line in new_dict.iteritems():
             candidate = self.detect_moved(obsolete_dict, line)
-            print "Candidate: "+str(candidate)
-            print "for line : "+str(line)
             if not candidate == None:
+                print "line moved: "+str(candidate)
+                print "to        : "+str(line)
+                print "-------------------------------------------------------------------------------------------------"
+
                 candidate.set_l10n_status("MOVED")
                 moved_cnt += 1
                 obsolete_cnt -= 1
+                new_cnt -= 1
                 line.keyid = candidate.keyid
-            ident.ident[candidate.get_identifier()] = line
-            del ident.ident[line.get_identifier()]
+                ident.ident[candidate.get_identifier()] = line
+                del ident.ident[line.get_identifier()]
  
         ident.write(self.default_ident_location)
-        print "Update:\nnew strings: "+str(new_cnt)+"\nmoved strings: "+str(moved_cnt)+"\nchanged strings: "+str(changed_cnt)+"\nuntouched strings: "+str(unchanged_cnt)+"\nobsolete strings: "+str(obsolete_cnt)+"\ntotal strings: "+str(new_cnt+changed_cnt+unchanged_cnt)
-    
+        #print "Update:"
+        #print "new strings: "+str(new_cnt)+"moved strings: "+str(moved_cnt)+"changed strings: "+str(changed_cnt)+
+        #print "untouched strings: "+str(unchanged_cnt)+"\nobsolete strings: "+str(obsolete_cnt)+"\ntotal strings: "+str(new_cnt+changed_cnt+unchanged_cnt)
+                
+        print "| new      : %(new_cnt)7d | moved   : %(moved_cnt)7d | changed: %(changed_cnt)7d |" % { "new_cnt": new_cnt, "moved_cnt": moved_cnt , "changed_cnt": changed_cnt }
+        print "| unchanged: %(unchanged_cnt)7d | obsolete: %(obsolete_cnt)7d | total  : %(total_cnt)7d |" % { "unchanged_cnt": unchanged_cnt, "obsolete_cnt": obsolete_cnt , "total_cnt": new_cnt+changed_cnt+unchanged_cnt }
+ 
     def detect_moved(self, ident, line):
         # filename, gid,lid,sid,hash
         pq = []
@@ -113,3 +137,24 @@
             return None
 
 
+    def clean_sdf2_data(self, cleanup_list):
+        print "Removeing obsolete strings from sdf2 data files"
+        default_localize_location =  os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n/source/*/localize.sdf2"
+        fh_list = []
+        for file in glob.glob(default_localize_location):
+            fh_list.append(open(file,"r+"))
+
+        for fh in fh_list:
+            sdf_dict = dict()
+            lines = [line.strip("\n") for line in fh.readlines()]
+            for line in lines:
+                sd = sdf2.Sdf2DataEntity()
+                sd.set_properties(line)
+                sdf_dict[sd.keyid] = sd
+            for item in cleanup_list:
+                if item.keyid in sdf_dict:
+                    del sdf_dict[item.keyid]
+            fh.truncate(0)
+            for val in sorted(sdf_dict.values(), key=lambda a: str(a)):
+                fh.write(str(val)+"\n")
+            fh.close()
diff --git a/l10ntools/scripts/tool/sdf2.py b/l10ntools/scripts/tool/sdf2.py
--- a/l10ntools/scripts/tool/sdf2.py
+++ b/l10ntools/scripts/tool/sdf2.py
@@ -104,15 +104,17 @@
     return sdfident
 
 def convert_sdfentity_to_sdf2data(line):
-	c_data = Sdf2DataEntity()
+    c_data = Sdf2DataEntity()
         #c_data.keyid = calc_keyid(line.project+line.source_file+line.lid+line.gid+line.platform+line.resource_type)         
         #c_data.source_hash = calc_hash(source_line.text+source_line.helptext+source_line.quickhelptext+source_line.title)
-        c_data.lang = line.lang          
-        c_data.text = line.text   
-        c_data.helptext = line.helptext       
-        c_data.quickhelptext = line.quickhelptext  
-        c_data.title = line.title          
- 	return c_data
+    c_data.source_hash = line.dummy1
+    c_data.keyid = line.keyid
+    c_data.lang = line.langid          
+    c_data.text = line.text   
+    c_data.helptext = line.helptext       
+    c_data.quickhelptext = line.quickhelptext  
+    c_data.title = line.title          
+    return c_data
 
 # PRE: sdf_list,sdf_source_list are lists of lang sdf_entities
 def convert_sdf_to_sdf2Data(sdf_source_list, sdf_list):
@@ -156,6 +158,7 @@
     sdfl = sdf.SdfEntity()
     sdfl.langid = c_data.lang 
     sdfl.text = c_data.text 
+    sdfl.dummy1 = c_ident.source_hash
     sdfl.helptext = c_data.helptext
     sdfl.quickhelptext = c_data.quickhelptext
     sdfl.title = c_data.title
@@ -179,9 +182,12 @@
 
 
 def convert_sdf2idententity_to_sdf(c_ident):
+    #print "ident:"
+    #print c_ident
     sdfl = sdf.SdfEntity()
     sdfl.project = c_ident.project 
     sdfl.source_file = c_ident.source_file
+    sdfl.dummy1 = c_ident.source_hash
     sdfl.resource_type = c_ident.resource_type 
     sdfl.gid = c_ident.keyid + ":" + c_ident.gid
     sdfl.lid = c_ident.lid 
@@ -191,6 +197,8 @@
     sdfl.helptext = c_ident.source_helptext 
     sdfl.quickhelptext = c_ident.source_quickhelptext 
     sdfl.title = c_ident.source_title
+    #print "sdf:"
+    #print sdfl
     return sdfl
 
 
