Language Studio utilizes a number of different approaches to capitalization depending on the language pair and client requirements. Some of the approaches use statistics that can be very good, but there is often a case where there is a source word that is output in the target language and you would like to match the capitalization from the source text. Often this can be important in languages such as Chinese or Japanese where English words could be in the output (for example brand names such as IBM or product names such as ProWave).


JS9 can be used to check any words in the output translation to see if the exist in the source and if found in the source to copy the capitalization.



 

 

 

 

//Language Studio Post-Processing Script
//Copyright 2008-2016 Omniscien Technologies Pte Ltd. All Rights Reserved.
//Use of any part of this script within non-Language Studio applications is not permited.

//Description:
//The script will match the capitalization of the source text when the same word is in the target.

var isDebug = 0; //SET TO 0 FOR PRODUCTION
var isExternalTest = 0; //SET TO 0 FOR PRODUCTION
var logText = "";
var iJSNo = 9; //The number of the JS Script - some logic can leverage the number
var sSourceLang = "en";
var sTargetLang = "ja";
var sLangPair = "";

//Detects if we are running in the Language Studio Enterprise runtime environment
var bIsDev = ((typeof AOSession) == "undefined");
if (bIsDev) {
    //We 
    isDebug = 1;
    isExternalTest = 1;
    //Set the paths to load the input files from for testing in Microsoft Visual Studio debugger. 
    var sTestInFilePathSource = currentPath() + "\\In" + iJSNo + "." + sSourceLang + ".source.txt";
    var sTestInFilePathTarget = currentPath() + "\\In" + iJSNo + "." + sTargetLang + ".target.txt";
    var sTestOutFilePath = currentPath() + "\\Out" + iJSNo + "." + sTargetLang + ".txt";
}

//Add prototypes if needed
if (!String.prototype.trim) {
    String.prototype.trim = function () {
        return this.replace(/^\s+|\s+$/g, '');
    }
}

function main(sAllSourceSegments, sAllTargetSegments) {

    //Initialize
    log("main (start)");
    initializeLang();

    //Process the all segments as 1 block of text
    //    sAllTargetSegments = processAllSegments(sAllSourceSegments, sAllTargetSegments);

    //Process each segment line one at a time
    sAllTargetSegments = processEachSegment(sAllSourceSegments, sAllTargetSegments);

    //Cleanup
    sAllTargetSegments = processFinalCleanup(sAllTargetSegments)

    log("main (exit)");

    // Return output
    return sAllTargetSegments;
}

//Process all segments together as one string
function processAllSegments(sAllSourceSegments, sAllTargetSegments) {
    log("processAllSegments (start)");

    //Your code here

    log("processAllSegments (exit)");

    return sAllTargetSegments;
}

//A crude form of tokenize that is only suitable for simple tasks of splitting up data for quick analysis. 
function basicTokenize(sIn) {
    var sOut = sIn;

    switch (sSourceLang.toLowerCase()) {
        case "zh":
        case "ja":
        case "ko":
            //Remove the Han text - to Tokenize these languages, need to word segment 
            sOut = sOut.replace(RegExp(/([\u4e00-\u9fa5]{1,})/gm), " "); //Replaces all Han text with a space
            //NOTE: Not to be used if you need the Han characters
            break;
    }
    sOut = sOut.replace(RegExp(/((?![\._,\-@])\p{P})/g), " $1 "); //Any punctuation except for .,_-@

    //Handle commas
    sOut = sOut.replace(RegExp(/([a-zA-Z][ ]{0,}),([ ]{0,}[a-zA-Z0-9])/g), "$1 , $2");
    sOut = sOut.replace(RegExp(/([a-zA-Z0-9][ ]{0,}),([ ]{0,}[a-zA-Z])/g), "$1 , $2");

    sOut = sOut.replace(RegExp(/([ ]{2,})/gi), " "); //Remove multiple spaces

    sOut = sOut.trim();

    return sOut;
}

function copySourceCapitalization(sSourceSegment, sTargetSegment) {
    var bChange = false;

    //Copy the source capitalization 
    sSourceSegment = basicTokenize(sSourceSegment);
    sTargetSegment = sTargetSegment.replace(RegExp(/([ ]{2,})/gmi), " ");
    sTargetSegment = sTargetSegment.trim();

    switch (sTargetLang.toLowerCase()) {
        case "zh":
        case "ko":
        case "ja":
        case "th":
        case "my":
        case "ar":
            //For non-romanized languages match the capitalization of all EN in the source
            var aTarget = sTargetSegment.split(' ');
            for (var i in aTarget) {
                //Find in source
                sWord = aTarget[i];
                //***Add your own language specific filters here if necessary.
                //Only process if there are letters
                if (sWord.match(new RegExp("[a-z]", "")) != null) {
                    //Look for target word in source text
                    var pattern = new RegExp("(^|[ ]|\\p{P})(" + sWord + ")($|[ ]|\\p{P})", "i");
                    var match = pattern.exec(sSourceSegment);
                    if (match != null) {
                        //Compare capitalization
                        if (match[2] != sWord) {
                            aTarget[i] = match[2];
                            bChange = true;
                        }
                    }
                }
            }
            if (bChange) {
                //There has been a change, so rebuid the target with the adjusted format
                sTargetSegment = aTarget.join(' ');
            }

            break;
        default: //Romanized languages
            var aTarget = sTargetSegment.split(' ');
            for (var i in aTarget) {
                //Find in source
                sTargetWord = aTarget[i];
                //***Add your own language specific filters here if necessary.
                //Only process if there are letters
                if (sTargetWord.match(/[A-Za-z]/)) {
                    //Look for target word in source text
                    var pattern = new RegExp("(^|[ ]|\\p{P})(" + sTargetWord + ")($|[ ]|\\p{P})", "i");
                    var match = pattern.exec(sSourceSegment);
                    if (match != null) {
                        //Compare capitalization
                        if (match[2] != sTargetWord) {
                            aTarget[i] = match[2];
                            bChange = true;
                        }
                    }
                }
            }
            if (bChange) {
                //If there has been a change, then rebuild the segment
                sTargetSegment = aTarget.join(' ');
            }
    }

    return sTargetSegment;
}

//Processes a single segment
function processSegment(sSourceSegment, sTargetSegment) {

    //Copy capitalization from the source text to the target when matching words are found in the target segment
    sTargetSegment = copySourceCapitalization(sSourceSegment, sTargetSegment);

    return sTargetSegment;
}

//Final functions to make sure that all data is valid and not small issues inserted such as BOM and multiple spaces
function processFinalCleanup(sAllTargetSegments) {
    log("processFinalCleanup (start)");

    try {
        //Safety checks - unlikely, but good practice to be sure.
        sAllTargetSegments = removeBOM(sAllTargetSegments); //Remove BOM
        sAllTargetSegments = sAllTargetSegments.replace(/([ ]{2,})/gm, " "); //Multiple spaces
        sAllTargetSegments = sAllTargetSegments.replace(/([ ]{0,}\n[ ]{0,})/gm, "\n"); //Trim each line

        //Your Code Here

    }
    catch (e) {
        logError(e);
    }

    log("processFinalCleanup (exit)");

    return sAllTargetSegments;
}

//-----------------------------------------------------------------
//Standard functions do not modify below this point
//-----------------------------------------------------------------

//Loop through each segment once at a time
function processEachSegment(sAllSourceSegments, sAllTargetSegments) {
    //Line break must be \n not \r\n
    var aSourceSegments = sAllSourceSegments.split("\n");
    var aTargetSegments = sAllTargetSegments.split("\n");
    for (var i in aSourceSegments) {
        //Process segment
        aTargetSegments[i] = processSegment(aSourceSegments[i], aTargetSegments[i]);
    }
    sAllTargetSegments = aTargetSegments.join("\n");
    return sAllTargetSegments;
}

//Removes a Byte Order Marker (BOM) from the start of data
function removeBOM(sIn) {
    var sBOM = "\xef\xbb\xbf";

    //Check for the BOM at the start of the data
    if (sIn.indexOf(sBOM) == 0) {
        sIn = sIn.substring(sBOM.length, sIn.length - sBOM.length);
    }
    return sIn;
}

//Counts how many time a string occurs in another string
function stringCount(sIn, sCount) {
    return sIn.split(sCount).length - 1;
}

//Sets up language variables
function initializeLang() {
    sSourceLang = sSourceLang.toLowerCase();
    sTargetLang = sTargetLang.toLowerCase();
    sLangPair = sSourceLang + '-' + sTargetLang;
}

//Writes a line to the log output
function log(sText) {
    if (isDebug == 1) {
        var now = new Date();
        logText += "\r\n" + formatLogDate(now) + "\t" + sText;
    }
}

function formatLogDate(dDate) {
    //zero-pad a single zero if needed
    var zp = function (val) {
        return (val <= 9 ? '0' + val : '' + val);
    }

    //zero-pad up to two zeroes if needed
    var zp2 = function (val) {
        return val <= 99 ? (val <= 9 ? '00' + val : '0' + val) : ('' + val);
    }

    return '' + dDate.getFullYear() + '-' + zp(dDate.getMonth() + 1) + '-' + zp(dDate.getDate()) + ' ' + zp(dDate.getHours()) + ':' + zp(dDate.getMinutes()) + ':' + zp(dDate.getSeconds()) + '.' + zp2(dDate.getMilliseconds());
}

function setScriptNo() {
    //Autodetect the script number from the name
    if (bIsDev) {
        var sNum = WScript.ScriptName.toString().replace(RegExp(/(.{0,})(\d{1,})(.{0,})/gmi), "$2");
        iJSNo = parseInt(sNum);
    }
    else {
        if (iJSNo == 0) {
            iJSNo = parseInt(AOSession.getValue("ScriptNo"));
        }
    }
}

//WINDOWS TEST FUNCTIONS - For use with WScript
if (isExternalTest) ExternalTest();

function ExternalTest() {
    if (isDebug) WScript.Echo('--Start Test--');

    //Read input from a file
    var sAllSourceSegments = fileToString(sTestInFilePathSource);
    var sAllTargetSegments = fileToString(sTestInFilePathTarget);

    //Normalize CRLF to \n
    sAllSourceSegments = sAllSourceSegments.replace(RegExp(/\r\n|\r/g), "\n");

    //Execute test as a normal pre-processing step would execute it. 
    var sOut = main(sAllSourceSegments, sAllTargetSegments);

    //Write the output to a file
    stringToFile(sTestOutFilePath, sOut);

    if (isDebug) {
        if (logText.length > 0) {
            WScript.Echo(logText);
        }
        WScript.Echo('--End Test--');
    }
}

function currentPath() {
    return new ActiveXObject("WScript.Shell").CurrentDirectory;
}

//Load the UTF-8 file into a string
function fileToString(sFilePath) {
    //Relies on  Microsoft ActiveX Data Objects (ADO) which much be installed - 
    var oStream = new ActiveXObject("ADODB.Stream");
    oStream.CharSet = "utf-8";
    oStream.Open;
    oStream.LoadFromFile(sFilePath);
    var sData = removeBOM(oStream.ReadText());
    oStream.Close();
    return sData;
}

//Write a string out to a UTF-8 file
function stringToFile(sFilePath, sData) {
    //Relies on  Microsoft ActiveX Data Objects (ADO) which much be installed - 
    var oStream = new ActiveXObject("ADODB.Stream");
    oStream.CharSet = "utf-8";
    oStream.Open
    oStream.WriteText(sData);
    oStream.SaveToFile(sFilePath, 2);
    oStream.Flush();
    oStream.Close();
}