# HG changeset patch
# User Ivo Hinkelmann <ihi@openoffice.org>
# Date 1294410619 -3600
# Node ID 79fa957269a21b33d3b89f73d8fa2ed1d130ef24
# Parent  6a2043940003c27c523e35a582335d2982f6ef5f
#i116350# Change sdf file format

diff --git a/l10ntools/scripts/tool/convert_to_sdf.py b/l10ntools/scripts/tool/convert_to_sdf.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/convert_to_sdf.py
@@ -0,0 +1,54 @@
+import sdf2, sdf, sys
+from optparse import OptionParser
+
+# convert the data only
+usage = "convert_to_sdf -ident <identfile> -data <datafile> -out <outputsdffile>"
+parser = OptionParser(usage=usage)
+parser.add_option("-i", "--ident", dest="ident", help="",default=False)
+parser.add_option("-d", "--data", dest="data", help="",default=False)
+parser.add_option("-o", "--out", dest="out", help="",default=False)
+
+(options, args) = parser.parse_args()
+
+if options.ident and options.data and options.out:
+    try:
+        fh = open(options.ident, "r")
+        ident_lines = [line.strip('\n') for line in fh.readlines()]
+    except IOError:
+        print "ERROR: Can not read file " + options.ident
+        sys.exit(13)
+    fh.close()
+    
+    mylist = list()
+    for line in ident_lines:
+        ie = sdf2.Sdf2IdentEntity()
+        ie.set_properties(line)
+        mylist.append(ie)
+    ident_lines = mylist
+
+    try:
+        fh = open(options.data, "r")
+        data_lines = [line.strip('\n') for line in fh.readlines()]
+    except IOError:
+        print "ERROR: Can not read file " + options.data
+        sys.exit(13)
+    fh.close()
+
+    mylist1 = list()
+    for line in data_lines:
+        de = sdf2.Sdf2DataEntity()
+        de.set_properties(line)
+        mylist1.append(de)
+    data_lines = mylist1
+
+    new_ident, new_data = sdf2.make_sdf2_list(ident_lines,data_lines)
+    sdf_lines = sdf2.convert_sdf2data_to_sdf(new_ident, new_data)
+    try:
+        fh = open(options.out, "w")
+        for line in sdf_lines:
+            fh.write(str(line)+"\n")
+    except IOError:
+        print "ERROR: Can not write file " + options.out
+        sys.exit(13)
+ 
+    sys.exit(0)
diff --git a/l10ntools/scripts/tool/fast_merge2.py b/l10ntools/scripts/tool/fast_merge2.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/fast_merge2.py
@@ -0,0 +1,158 @@
+from optparse import OptionParser
+import sdffile, sdf2
+import sys, os , cStringIO
+
+
+def make_path(merge_dir, line):
+    dir_str = merge_dir+"/"+line.project+"/"+line.source_file
+    dir_str = dir_str.replace("\\","/") 
+    x = dir_str.split("/")[0:-2]
+    dir_str = "/".join(x) 
+    return dir_str
+
+def mkdir(dir_str):
+    try:
+        os.makedirs(dir_str)
+    except:
+        pass
+
+def process_ident(ident_file, merge_dir):
+    old_str = ""
+    f = None
+    for line in ident_file:
+        dir_str = make_path(merge_dir, line)        
+        mkdir(dir_str)        
+        dir_str += "/ident.sdf2"
+        if old_str != dir_str:
+            old_str = dir_str
+            if f != None:
+                f.close()
+            try:
+                os.unlink(dir_str)
+            except:
+                pass
+            try:
+                f = open(dir_str, "wa")
+            except IOError:
+                print "ERROR: Can not write file " + dir_str
+                sys.exit(-1)
+        f.write(str(line)+"\n")
+    if f != None:
+        f.close() 
+
+def process_sdf2_data(sdf2_list, merge_dir, fh_dict, ident_dict):
+    #print sdf2_list
+    for sdf2_file in sdf2_list:
+        data = read_sdf2_data_file(sdf2_file)
+        for keyid,line in data.iteritems():        
+            #link ident + data
+            fh = get_fh(ident_dict[keyid], merge_dir, fh_dict)
+            fh.write(str(line)+"\n")
+        
+def write(fh_dict):
+    for file,fh in fh_dict.iteritems():
+        try:
+            f = open(file,"wa")
+        except:
+            print "Can not write to file "+file
+            sys.exit(-1)
+        print "Writing file "+file
+        f.write(fh.getvalue())    
+        f.close()
+        fh.close()
+        fh_dict[file] = cStringIO.StringIO()
+
+def get_fh(line, merge_dir, fh_dict):
+    dir_str = make_path(merge_dir, line)
+    dir_str += "/data.sdf2"  
+    if dir_str in fh_dict:
+        return fh_dict[dir_str]
+    else:
+        fh = cStringIO.StringIO()    
+        fh_dict[dir_str] = fh
+    return fh
+
+def read_file(file):
+    lines = []
+    try:
+        f = open(file, "r")
+        lines = [line.strip('\n') for line in f.readlines()]
+    except IOError:
+        print "ERROR: Can not read file " + file
+        sys.exit(-1)
+    else:
+        f.close()
+    return lines
+
+def read_sdf2_data_file(sdf2_file):
+    sdf2_data = dict()
+    print "Reading file "+sdf2_file
+    lines = read_file(sdf2_file)
+    for line in lines:
+        de = sdf2.Sdf2DataEntity()
+        de.set_properties(line)
+        sdf2_data[de.keyid] = de
+    return sdf2_data
+
+def read_ident_file(sdf2_ident_file):
+    ident_file = []
+    ident_dict = dict()
+    lines = read_file(sdf2_ident_file)
+    for line in lines:
+        ide = sdf2.Sdf2IdentEntity()
+        ide.set_properties(line)
+        ident_file.append(ide)
+        ident_dict[ide.keyid] = ide
+    return ident_file, ident_dict
+
+def read_sdf_file_list(sdf2_files):
+    file_list = []
+    lines = read_file(sdf2_files)
+    for line in lines:
+        for file in line.split(" "):
+            file_list.append(file)
+    return file_list
+
+def parse_options():
+    parser = OptionParser()
+    parser.add_option("-s", "--sdf2files",       dest="sdf2files",       metavar="FILE", help="")
+    parser.add_option("-i", "--sdf2identfile",   dest="sdf2identfile",   metavar="FILE", help="")
+    parser.add_option("-d", "--mergedir",        dest="mergedir",         metavar="DIR",  help="")
+
+    (options, args) = parser.parse_args()
+
+    is_valid = lambda x: x != None and len(x) > 0
+
+    if is_valid(options.sdf2files) and is_valid(options.mergedir) and is_valid(options.sdf2identfile):
+        return options.sdf2files, options.mergedir, options.sdf2identfile
+    else:
+        usage()
+        sys.exit(-1)
+    
+def usage():
+    return "Usage: fast_merge --sdf2files <file containing sdf file names> --sdf2identfile <identfile> --mergedir <directory>\n"
+
+
+MAX_OPEN_FILES = 35 
+sdf2_files, merge_dir, sdf2_ident_file = parse_options()
+sdf2_file_list = read_sdf_file_list(sdf2_files)
+ident_file, ident_dict = read_ident_file(sdf2_ident_file)
+
+sdf2_chunks = []
+fh_dict = dict()
+for x in range(0, int(round((len(sdf2_file_list) / MAX_OPEN_FILES) + 0.5))):
+    sdf2_chunks.append(list())
+
+list_counter = -1
+for x in range(0,len(sdf2_file_list)):
+    if x % MAX_OPEN_FILES == 0:
+        list_counter += 1
+    sdf2_chunks[list_counter].append(sdf2_file_list[x])
+
+process_ident(ident_file, merge_dir)
+for part in sdf2_chunks:
+    process_sdf2_data(part, merge_dir, fh_dict, ident_dict)
+    write(fh_dict)    
+sys.exit(0)
+
+
diff --git a/l10ntools/scripts/tool/init_sdf2.py b/l10ntools/scripts/tool/init_sdf2.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/init_sdf2.py
@@ -0,0 +1,67 @@
+import sdf2, sdf, sys
+from optparse import OptionParser
+
+# convert the data only
+usage = "init_sdf2 --input <input_sdf_file> --ref <input_en-US_sdf_file> --ident <outputidentfile> -data <outputdatafile> "
+parser = OptionParser(usage=usage)
+parser.add_option("-i", "--ident", dest="ident", help="",default=False)
+parser.add_option("-d", "--data", dest="data", help="",default=False)
+parser.add_option("-r", "--ref", dest="ref", help="",default=False)
+parser.add_option("-f", "--input", dest="input", help="",default=False)
+
+(options, args) = parser.parse_args()
+
+if options.input and options.ref:
+    try:
+        fh = open(options.input, "r")
+        sdf_data_lines = [line.strip('\n') for line in fh.readlines()]
+    except IOError:
+        print "ERROR: Can not read file " + options.input
+        sys.exit(13)
+    fh.close()
+    
+    mylist = list()
+    for line in sdf_data_lines:
+        se = sdf.SdfEntity()
+        se.set_properties(line)
+        mylist.append(se)
+    sdf_data_lines = mylist
+
+    try:
+        fh = open(options.ref, "r")
+        sdf_source_lines = [line.strip('\n') for line in fh.readlines()]
+    except IOError:
+        print "ERROR: Can not read file " + options.ref
+        sys.exit(13)
+    fh.close()
+    
+    mylist1 = list()
+    for line in sdf_source_lines:
+        se = sdf.SdfEntity()
+        se.set_properties(line)
+        mylist1.append(se)
+    sdf_source_lines = mylist1
+
+    if options.data:
+        #new_source, new_data = sdf2.make_sdf_list(sdf_source_lines,sdf_data_lines)
+        sdf2_data = sdf2.convert_sdf_to_sdf2Data(sdf_source_lines, sdf_data_lines)
+        try:
+            fh = open(options.data, "w")
+            for line in sdf2_data:
+                fh.write(str(line)+"\n")
+        except IOError:
+            print "ERROR: Can not write file " + options.data
+            sys.exit(13)
+        fh.close()
+    
+    if options.ident:
+        sdf2_ident= sdf2.convert_sdf_to_sdf2Ident(sdf_source_lines)
+        try:
+            fh = open(options.ident, "w")
+            for line in sdf2_ident:
+                fh.write(str(line)+"\n")
+        except IOError:
+            print "ERROR: Can not write file " + options.ident
+            sys.exit(13)
+        fh.close()
+    sys.exit(0)
diff --git a/l10ntools/scripts/tool/l10n.py b/l10ntools/scripts/tool/l10n.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/l10n.py
@@ -0,0 +1,28 @@
+import sys, l10nupdate, l10nverify, l10nstatus
+
+def usage():
+    print "help!"
+
+if not len(sys.argv) == 1:
+    cmd = sys.argv[1]
+else:
+    usage()
+    sys.exit(13)
+
+if(cmd == "update"):
+    l10nupdate.run()
+elif(cmd == "verify"):
+    l10nverify.run()
+elif(cmd == "status"):
+    l10nstatus.run(sys.argv)
+elif(cmd == "new"):
+    l10nstatus.new(sys.argv)
+elif(cmd == "changed"):
+    l10nstatus.changed(sys.argv)
+elif(cmd == "translated"):
+    l10nstatus.translated(sys.argv)
+elif(cmd == "help"):
+    help()
+
+
+
diff --git a/l10ntools/scripts/tool/l10nstatus.py b/l10ntools/scripts/tool/l10nstatus.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/l10nstatus.py
@@ -0,0 +1,120 @@
+import os, sys, glob, sdf2
+
+def run(opts):
+    s = l10nstatus(opts)
+    s.exe()
+
+def translated(opts):
+    s = l10nstatus(opts)
+    s.exe("translated")
+
+def new(opts):
+    s = l10nstatus(opts)
+    s.exe("new")
+
+def changed(opts):
+    s = l10nstatus()
+    s.exe("changed")
+
+class l10nstatus():
+    NEW = "new"
+    CHANGED = "changed"
+    TRANSLATED = "translated"
+
+    def __init__(self, opts):
+        self.lines = list()
+        self.ident = dict()
+        self.opts = opts
+
+    def exe(self, cmd=""):
+        use_lang = ""
+        default_ident_location = os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n2/source/ident.sdf2"
+        if len(self.opts) == 2:
+            default_location = os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n2/source/*/localize.sdf2" 
+        elif len(self.opts) == 3:
+            use_lang = self.opts[2]
+            lang_list = use_lang.split(",")    
+        try:
+            fh = open(default_ident_location, "r")
+            tmplines = [line.strip('\n') for line in fh.readlines()]
+        except IOError:
+            print "Can not open file "+file
+            sys.exit(-1)
+         
+        status = L10nStatus()
+        self.read_ident_data(default_ident_location)
+
+        if use_lang == "":
+            sdf_data_list = glob.glob(default_location)
+        else:
+            sdf_data_list = list()
+            for lang in lang_list:
+                sdf_data_list.append(os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n2/source/"+lang+"/localize.sdf2")
+                        
+        for file in sdf_data_list:
+            mylines = list()
+            try:
+                fh = open(file, "r")
+                lines = [line.strip('\n') for line in fh.readlines()]
+            except IOError:
+                print "Can not openfile "+file
+            lang = file.split("/")
+            lang = lang[-2]
+            
+            status.new[lang] = dict()
+            status.translated[lang] = dict()
+            status.changed[lang] = dict()
+            for line in lines:
+                id = sdf2.Sdf2DataEntity()
+                id.set_properties(line)
+                if id.keyid in self.ident:
+                    ref_line = self.ident[id.keyid]
+                    if ref_line.source_hash == id.source_hash:
+                        status.translated[lang][id.keyid] = id
+                    else:
+                        status.changed[lang][id.keyid] = id
+            for line in self.lines: 
+                if not line.keyid in status.translated[lang] :
+                    status.new[lang][line.keyid] = line
+        
+            self.print_status(status, cmd, lang)
+            
+            # Cleanup
+            del status.new[lang]
+            del status.translated[lang]
+            del status.changed[lang]
+    
+    def print_status(self, status, cmd, lang):
+        if cmd == "translated":
+            for x in status.translated[lang].values():
+                print str(x)
+        elif cmd == "new":
+            for x in status.new[lang].values():
+                print str(x)
+        elif cmd == "changed":
+            for x in status.changed[lang].values():
+                print str(x)
+        else:
+            print "%(lang)7s" % { "lang": lang} +":"+"%(num)7d / %(per)3g" % {"num": len(status.new[lang].values()), "per": len(status.new[lang].values()) *100 / len(self.lines)} + "% new "  \
+                  +"%(num1)7d / %(per1)3g" % {"num1": len(status.translated[lang].values()), "per1": len(status.translated[lang].values()) *100 / len(self.lines)} + "% translated " \
+                  +"%(num2)7d / %(per2)3g" % {"num2": len(status.changed[lang].values()), "per2": len(status.changed[lang].values()) *100 / len(self.lines) } +"% changed "
+ 
+    def read_ident_data(self, default_ident_location):
+        try:
+            fh = open(default_ident_location, "r")
+            tmplines = [line.strip('\n') for line in fh.readlines()]
+        except IOError:
+            print "Can not open file "+file
+            sys.exit(-1)
+            
+        for line in tmplines:
+            ie = sdf2.Sdf2IdentEntity()
+            ie.set_properties(line)
+            self.ident[ie.keyid] = ie
+            self.lines.append(ie)
+
+class L10nStatus():
+    ident = dict()
+    new = dict()
+    changed = dict()
+    translated = dict()
diff --git a/l10ntools/scripts/tool/l10nupdate.py b/l10ntools/scripts/tool/l10nupdate.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/l10nupdate.py
@@ -0,0 +1,69 @@
+import tempfile, sys, subprocess, sdf, os, sdffile
+
+# run localize
+# add all new strings to sourcedb
+
+def run():
+    a = l10nupdate()
+    a.update_ident_file()
+
+class l10nupdate:
+    # fix me ;)
+    default_ident_location = os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n2/source/ident.sdf2"
+    
+    def __init__(self):
+        pass
+    
+    def run_localize(self):
+        tmpfile = "extract"
+#        tmpfile = tempfile.mkstemp()[1]
+#        localize = "localize_sl -e -l all -f " + tmpfile
+
+#        print "Executing " + localize
+#        bufsize = -1
+#        try:
+#            p = subprocess.Popen(localize, bufsize, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)       
+#            stdout_value, stderr_value = p.communicate()
+#        except OSError, e:
+#            print  "Error in Util.exec("+command+")\n OSError: Command not found?!" + str(e)
+#        except ValueError, e:
+#            print "Error in Util.exec("+command+")\n ValueError: false / broken command switches?!" + str(e)
+        
+#        if(p.returncode != 0):
+#            print "ERROR!!!"
+#            print stdout_value
+#            print stderr_value
+        
+        try:
+            f = open(tmpfile, "r")
+            lines = [line.strip('\n') for line in f.readlines()]
+        except IOError:
+            pass
+        
+        sdffile = list()
+        for line in lines:
+            se = sdf.SdfEntity()
+            se.set_properties(line)
+            sdffile.append(se)
+        
+        #os.remove(tmpfile)
+        return sdffile
+    
+    def update_ident_file(self):
+        sdf_file = self.run_localize()
+        ident = sdffile.IdentFile()
+        ident.read(self.default_ident_location)
+        old_cnt, new_cnt, changed_cnt, moved_cnt = 0, 0, 0, 0
+        for line in sdf_file:
+            status = ident.add_sdf(line)
+            if status == "OLD":
+                old_cnt += 1
+            elif status == "NEW":
+                new_cnt += 1
+            elif status == "CHANGED":
+                changed_cnt += 1
+            elif status == "MOVED":
+                moved_cnt += 1
+        ident.write(self.default_ident_location)
+        print "Update:\nnew strings: "+str(new_cnt)+"\nmoved strings: "+str(moved_cnt)+"\nchanged strings: "+str(changed_cnt)+"\nuntouched strings: "+str(old_cnt)+"\ntotal strings: "+str(new_cnt+changed_cnt+old_cnt)
+
diff --git a/l10ntools/scripts/tool/l10nverify.py b/l10ntools/scripts/tool/l10nverify.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/l10nverify.py
@@ -0,0 +1,29 @@
+import os, sdf2
+# check that there is no dupe identifier
+
+def run():
+    v = l10nverify()
+    v.read()
+
+class l10nverify():
+
+    def __init__(self):
+        self.lines = list()
+        
+    def read(self):
+        default_ident_location = os.environ["SOURCE_ROOT"]+"/"+os.environ["WORK_STAMP"]+"/ooo/l10n2/source/ident.sdf2"
+        
+        try:
+            f = open(default_ident_location, "r")
+            lines = [line.strip('\n') for line in f.readlines()]
+        except IOError:
+            pass
+        keyids = dict();
+        for line in lines:
+            ie = sdf2.Sdf2IdentEntity()
+            ie.set_properties(line)
+            #print str(ie)
+            if ie.keyid in keyids:
+                print "Found dupe:\n"+str(keyids[ie.keyid])+"\n"+str(ie)
+            else:
+                keyids[ie.keyid] = ie
diff --git a/l10ntools/scripts/tool/sdf2.py b/l10ntools/scripts/tool/sdf2.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/sdf2.py
@@ -0,0 +1,301 @@
+import zlib, hashlib, const, sdf
+
+#usage = "sdfconvert -ident <identfile> -data <datafile> -out <outputsdffile>"
+#parser = OptionParser(usage=usage)
+#parser.add_option("-m", "--master", action="store_true", dest="master", help="run master controller",default=False)
+#parser.add_option("-c", "--client", action="store_true", dest="client", help="run client with syncronization",default=False)
+#if options.master and options.client:
+#    parser.error("Run either in master or in client or in standalone mode")
+#    sys.exit(1)
+
+
+def calc_hash(text):
+    text = text.replace("\t"," ")
+    #print "calc hash over "+text
+    m = hashlib.md5()
+    m.update(text)
+    return m.hexdigest()
+
+def calc_keyid(text):
+    return __make_id(text)
+
+def __make_id(text):
+    text = text.replace("\t"," ")
+    #print "calc keyid over "+text
+    if(text == "scp2 source\\ooo\\directory_ooo.ulf LngText STR_DIR_KAPITEL   "): 
+        return "keyid1"
+    hash = zlib.crc32(text) & 0xffffffff
+    return __make_number(hash)
+
+def __make_number(h):
+    #                   1         2         3         4
+    #          1234567890123456789012345678901234567890
+    symbols = "0123456789abcdefghijklmnopqrstuvwxyz+-[=]ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    order = len(symbols)
+    result = ""
+    while( len(result) < 6 ):
+        result += str(symbols[int(h) % int(order)])
+        h = int( h / order )
+    if h > 0:  
+        print "make_number failed because number is too big (this cannot be so this is a strange error)"
+    return result[::-1]
+           
+
+# read both list, make them zipable by dropping data's without idents
+def make_sdf_list(sdf_source_list, sdf_data_list):
+    source_list = list()
+    data_list = list()
+    
+    data_hash = dict()
+    for data_line in sdf_data_list:
+        id =  data_line.project+data_line.gid+data_line.lid+data_line.source_file+data_line.resource_type+data_line.platform
+        data_hash[id] = data_line
+
+   
+    source_hash = dict()
+    for source_line in sdf_source_list:
+        id =  source_line.project+source_line.gid+source_line.lid+source_line.source_file+source_line.resource_type+source_line.platform
+        if(id in data_hash):
+            source_list.append(source_line)
+            data_list.append(data_hash[id])
+    
+    return source_list, data_list
+
+def make_sdf2_list(sdf2_ident_list, sdf2_data_list):
+    source_list = list()
+    data_list = list()
+    
+    data_hash = dict()
+    for data_line in sdf2_data_list:
+        id =  data_line.keyid
+        data_hash[id] = data_line
+
+    source_hash = dict()
+    for source_line in sdf2_ident_list:
+        id =  source_line.keyid
+        if(id in data_hash):
+            source_list.append(source_line)
+            data_list.append(data_hash[id])
+    
+    return source_list, data_list
+  
+
+# PRE: sdf_list is a list of en-US sdf_entities
+def convert_sdf_to_sdf2Ident(sdf_list):
+    sdfident = list()
+
+    for line in sdf_list:
+        c_ident = Sdf2IdentEntity()
+        c_ident.project = line.project
+        c_ident.source_file = line.source_file
+        c_ident.resource_type = line.resource_type
+        c_ident.gid = line.gid           
+        c_ident.lid = line.lid           
+        c_ident.sid = line.platform           
+        c_ident.lang = line.langid          
+        c_ident.keyid = calc_keyid(line.project+line.source_file+line.lid+line.gid+line.platform+line.resource_type)
+        c_ident.source_hash = calc_hash(line.text+line.helptext+line.quickhelptext+line.title)
+        c_ident.source_text = line.text   
+        c_ident.source_helptext = line.helptext       
+        c_ident.source_quickhelptext = line.quickhelptext  
+        c_ident.source_title = line.title   
+        sdfident.append(c_ident)
+
+    return sdfident
+
+# PRE: sdf_list,sdf_source_list are lists of lang sdf_entities
+def convert_sdf_to_sdf2Data(sdf_source_list, sdf_list):
+    sdfdata = list()
+
+    mydict = dict()
+    for line in sdf_source_list:
+        mydict[line.get_id2()] = line
+
+    for line in sdf_list:
+        if line.get_id2() in mydict:
+            source_line = mydict[line.get_id2()]
+            c_data = Sdf2DataEntity()
+            c_data.keyid = calc_keyid(line.project+line.source_file+line.lid+line.gid+line.platform+line.resource_type)         
+            c_data.source_hash = calc_hash(source_line.text+source_line.helptext+source_line.quickhelptext+source_line.title)
+            c_data.lang = line.langid          
+            c_data.text = line.text   
+            c_data.helptext = line.helptext       
+            c_data.quickhelptext = line.quickhelptext  
+            c_data.title = line.title          
+            sdfdata.append(c_data)
+        else:
+            #print "Can not find match for: "+str(line)
+            pass
+
+#    for line,source_line in zip(sdf_list,sdf_source_list):
+#        c_data = Sdf2DataEntity()
+#        c_data.keyid = calc_keyid(line.project+line.source_file+line.lid+line.gid+line.platform+line.resource_type)         
+#        c_data.source_hash = calc_hash(source_line.text+source_line.helptext+source_line.quickhelptext+source_line.title)
+#        c_data.lang = line.langid          
+#        c_data.text = line.text   
+#        c_data.helptext = line.helptext       
+#        c_data.quickhelptext = line.quickhelptext  
+#        c_data.title = line.title          
+#        sdfdata.append(c_data)
+
+    return sdfdata
+
+
+def convert_sdf2data_to_sdf(sdf2_ident_list, sdf2_data_list):
+    sdf_list = list()
+
+    for c_ident,c_data in zip(sdf2_ident_list,sdf2_data_list):
+        sdfl = sdf.SdfEntity()
+        sdfl.langid = c_data.lang 
+        sdfl.text = c_data.text 
+        sdfl.helptext = c_data.helptext
+        sdfl.quickhelptext = c_data.quickhelptext
+        sdfl.title = c_data.title
+        sdfl.project = c_ident.project 
+        sdfl.source_file = c_ident.source_file
+        sdfl.resource_type = c_ident.resource_type 
+        sdfl.gid = c_ident.gid
+        sdfl.lid = c_ident.lid 
+        sdfl.platform = c_ident.sid 
+        sdf_list.append(sdfl)
+    
+    return sdf_list
+
+def convert_sdf2ident_to_sdf(sdf2_ident_list):
+    sdf_list = list()
+
+    for c_ident in sdf2_ident_list:
+        sdfl = sdf.SdfEntity()
+        sdfl.project = c_ident.project 
+        sdfl.source_file = c_ident.source_file
+        sdfl.resource_type = c_ident.resource_type 
+        sdfl.gid = c_ident.gid
+        sdfl.lid = c_ident.lid 
+        sdfl.sid = c_ident.sid 
+        sdfl.lang = c_ident.lang 
+        sdfl.text = c_ident.source_text
+        sdfl.helptext = c_ident.source_helptext 
+        sdfl.quickhelptext = c_ident.source_quickhelptext 
+        sdfl.title = c_ident.source_title
+        sdf_list.append(sdfl)
+
+    return sdf_list   
+
+
+class Sdf2DataEntity: 
+    KEYID_POS           = 0
+    SOURCE_HASH_POS     = 1
+    LANG_POS            = 2
+    TEXT_POS            = 3
+    HELPTEXT_POS        = 4
+    QUICKHELPTEXT_POS   = 5
+    TITLE_POS           = 6
+        
+    def __init__(self, keyid="", source_hash="", lang="", text="", helptext="", quickhelptext="", title=""):
+        self.keyid          = keyid
+        self.source_hash    = source_hash
+        self.lang           = lang
+        self.text           = text;
+        self.helptext       = helptext;
+        self.quickhelptext  = quickhelptext;
+        self.title          = title;
+
+    def set_properties(self, line):
+        splitted = line.split("\t")
+        if len(splitted) == 7:
+            self.keyid          = splitted[self.KEYID_POS]             
+            self.source_hash    = splitted[self.SOURCE_HASH_POS]     
+            self.lang           = splitted[self.LANG_POS]         
+            self.text           = splitted[self.TEXT_POS]
+            self.helptext       = splitted[self.HELPTEXT_POS]
+            self.quickhelptext  = splitted[self.QUICKHELPTEXT_POS]
+            self.title          = splitted[self.TITLE_POS]
+    
+    def __str__(self):
+        return ''.join([self.keyid, "\t", self.source_hash, "\t",  self.lang, "\t",  self.text, "\t",
+            self.helptext, "\t",  self.quickhelptext, "\t", self.title])
+ 
+class Sdf2IdentEntity: 
+    PROJECT_POS         = 0
+    SOURCE_FILE_POS     = 1
+    RESOURCE_TYPE_POS   = 2
+    GID_POS             = 3
+    LID_POS             = 4
+    SID_POS             = 5
+    LANG_POS            = 6
+    KEYID_POS           = 7
+    SOURCE_HASH_POS     = 8
+    SOURCE_TEXT_POS            = 9
+    SOURCE_HELPTEXT_POS        = 10
+    SOURCE_QUICKHELPTEXT_POS   = 11
+    SOURCE_TITLE_POS           = 12
+ 
+    def __init__(self, project="", source_file="", resource_type="", gid="", lid="", sid="", lang="", 
+                       keyid="", source_hash="", source_text="", source_helptext="", source_quickhelptext="", source_title=""):
+        self.project        = project
+        self.source_file    = source_file
+        self.resource_type  = resource_type
+        self.gid            = gid
+        self.lid            = lid
+        self.sid            = sid
+        self.lang           = lang
+        self.keyid          = keyid
+        self.source_hash    = source_hash
+        self.source_text    = source_text
+        self.source_helptext       = source_helptext
+        self.source_quickhelptext  = source_quickhelptext
+        self.source_title          = source_title
+
+    def set_properties(self, line):
+        splitted = line.split("\t")
+        if len(splitted) == 13:
+            self.project        = splitted[self.PROJECT_POS]             
+            self.source_file    = splitted[self.SOURCE_FILE_POS]     
+            self.resource_type  = splitted[self.RESOURCE_TYPE_POS] 
+            self.gid            = splitted[self.GID_POS]             
+            self.lid            = splitted[self.LID_POS]             
+            self.sid            = splitted[self.SID_POS]             
+            self.lang           = splitted[self.LANG_POS]         
+            self.keyid          = splitted[self.KEYID_POS]         
+            self.source_hash    = splitted[self.SOURCE_HASH_POS]         
+            self.source_text    = splitted[self.SOURCE_TEXT_POS]         
+            self.source_helptext= splitted[self.SOURCE_HELPTEXT_POS]         
+            self.source_quickhelptext = splitted[self.SOURCE_QUICKHELPTEXT_POS]         
+            self.source_title   = splitted[self.SOURCE_TITLE_POS]         
+    
+    def __str__(self):
+        return ''.join([self.project, "\t", self.source_file, "\t", self.resource_type, "\t", self.gid, "\t",  
+            self.lid, "\t", self.sid, "\t", self.lang, "\t", self.keyid, "\t", self.source_hash, "\t",  
+            self.source_text, "\t", self.source_helptext, "\t", self.source_quickhelptext, "\t", self.source_title])
+    
+    def get_identifier(self):
+        return ''.join([self.project, "\t", self.source_file, "\t", self.resource_type, "\t", self.gid, "\t",  
+            self.lid, "\t", self.sid])
+         
+# testdriver
+text = "ivo is testing this"
+
+#print "md5:"
+#print calc_hash(text)
+#print "keyid:"
+#print calc_keyid(text)
+
+#sdf_e = sdf.SdfEntity()
+#sdf_e.set_properties("avmedia\tsource\\framework\\mediacontrol.src\t0\tstring\tAVMEDIA_STR_PAUSE\t\t\t\t0\ten-US\tPause\t\t\t\t2002-02-02 02:02:02")
+#mylist = list()
+#mylist.append(sdf_e)
+#retlist = convert_sdf_to_sdf2Ident(mylist)
+#print retlist[0]
+
+#sdf_e1 = sdf.SdfEntity()
+#sdf_e1.set_properties("avmedia\tsource\\framework\\mediacontrol.src\t0\tstring\tAVMEDIA_STR_PAUSE\t\t\t\t0\tde\tPause\t\t\t\t2002-02-02 02:02:02")
+#mylist1 = list()
+#mylist1.append(sdf_e1)
+#mylist2 = list()
+#sdf_e2 = sdf.SdfEntity()
+#sdf_e2.set_properties("avmedia\tsource\\framework\\mediacontrol.src\t0\tstring\tAVMEDIA_STR_PAUSE\t\t\t\t0\ten-US\tPause\t\t\t\t2002-02-02 02:02:02")
+#mylist2.append(sdf_e2)
+# source_list,target_list
+#retlist1 = convert_sdf_to_sdf2Data(mylist2, mylist1)
+#print retlist1[0]
+
diff --git a/l10ntools/scripts/tool/sdffile.py b/l10ntools/scripts/tool/sdffile.py
new file mode 100644
--- /dev/null
+++ b/l10ntools/scripts/tool/sdffile.py
@@ -0,0 +1,120 @@
+import sys, sdf2, heapq
+
+class IdentFile:
+    
+    def __init__(self):
+        self.lines = list()
+        self.ident = dict()
+        self.used_keyids = dict()
+        
+    def read(self, file):
+        try:
+            fh = open(file, "r")
+            tmplines = [line.strip('\n') for line in fh.readlines()]
+        except IOError:
+            print "Can not open file "+file
+            sys.exit(-1)
+            
+        # Update ident hash    
+        for line in tmplines:
+            ie = sdf2.Sdf2IdentEntity()
+            ie.set_properties(line)
+            self.ident[ie.get_identifier()] = ie
+            self.used_keyids[ie.keyid] = "1"
+            self.lines.append(ie)
+            
+    def write(self, file):
+        try:
+            fh = open(file, "w")
+            mylist = list()
+            self.rehash()
+            for line in self.ident.values():
+                mylist.append(str(line))
+            mylist.sort(key=str.lower)
+            for line in mylist:
+                fh.write(line+"\n")
+        except IOError:
+            print "Can not write to file "+file
+            sys.exit(-1)
+
+    def add_sdf(self, line):
+        c_ident = sdf2.Sdf2IdentEntity()
+        c_ident.project = line.project
+        c_ident.source_file = line.source_file
+        c_ident.resource_type = line.resource_type
+        c_ident.gid = line.gid           
+        c_ident.lid = line.lid           
+        c_ident.sid = line.platform           
+        c_ident.lang = line.langid          
+        #c_ident.keyid = sdf2.calc_keyid(line.project+line.source_file+line.lid+line.gid+line.platform+line.resource_type)
+        c_ident.source_hash = sdf2.calc_hash(line.text+line.helptext+line.quickhelptext+line.title)
+        c_ident.source_text = line.text   
+        c_ident.source_helptext = line.helptext       
+        c_ident.source_quickhelptext = line.quickhelptext  
+        c_ident.source_title = line.title   
+        return self.add(c_ident)
+
+    # line is a Sdf2IdentEntity
+    def add(self, line):
+        status = ""
+        if self.ident.has_key(line.get_identifier()):
+            ref_line = self.ident[line.get_identifier()]
+            keyid = ref_line.keyid
+            status = "OLD"
+            ref_l10n = ref_line.source_text+ref_line.source_helptext+ref_line.source_quickhelptext+ref_line.source_title
+            new_l10n = line.source_text+line.source_helptext+line.source_quickhelptext+line.source_title
+            if not ref_l10n == new_l10n:
+                status = "CHANGED"
+        else:
+            # new string
+            #line.source_hash = sdf2.calc_hash(line.source_text+line.source_helptext+line.source_quickhelptext+line.source_title)
+            status = "NEW"
+            candidate = self.detect_moved(self.lines, line)
+            if not candidate == None:
+                print "Move detected: "+str(line)
+                print "Candidate:     "+str(candidate)
+                keyid = candidate.keyid
+                del self.ident[candidate.get_identifier()]
+                status = "MOVED"
+            else:
+                keyid = sdf2.calc_keyid(line.project+line.source_file+line.lid+line.gid+line.sid+line.resource_type)
+                cnt = 0
+                while(keyid in self.used_keyids):
+                    cnt = cnt + 1
+                    keyid = sdf2.calc_keyid(str(line.project)+str(line.source_file)+str(line.lid)+str(line.gid)+str(line.sid)+str(line.resource_type)+str(cnt))
+        line.keyid = keyid
+        self.used_keyids[keyid] = 1
+        self.ident[line.get_identifier()] = line
+        return status
+
+    def detect_moved(self, ident, line):
+        # filename, gid,lid,sid,hash
+        pq = []
+        
+        for ident_line in ident:
+            val = 32
+            if ident_line.source_hash == line.source_hash:
+                if ident_line.project == line.project:
+                    val /= 2
+                if ident_line.source_file == line.source_file:
+                    val /= 2
+                if ident_line.gid == line.gid:
+                    val /= 2
+                if ident_line.lid == line.lid:
+                    val /= 2
+                if ident_line.sid == line.sid:
+                    val /= 2
+                heapq.heappush(pq, (val, ident_line))
+        if len(pq) > 0:
+            val, candidate = heapq.heappop(pq)
+            # At least three equal attributes found! 
+            if val <= 4:
+                return candidate
+            else:
+                return None
+        else:
+            return None
+
+    def rehash(self):
+        for line in self.ident.values():
+            line.source_hash = sdf2.calc_hash(line.source_text+line.source_helptext+line.source_quickhelptext+line.source_title)
