|
|
 next
|
  Subject: java/lang/OutOfMemoryError Running large data file
Author: dulmini s
Date: 28 Feb 2013 12:32 AM
|
If try to run using Stylus studio X14(Build 1814, Data Direct xQuery 5.0.0.0763 )I get beow error
java.lang.OutOfMemoryError: Java heap space
at com.ddtek.xquery.saxon.tinytree.TinyTree.ensureNodeCapacity(TinyTree.java:254)
at com.ddtek.xquery.saxon.tinytree.TinyTree.addNode(TinyTree.java:353)
at com.ddtek.xquery.saxon.tinytree.TinyBuilder.startElement(TinyBuilder.java:207)
at com.ddtek.xquery.saxon.event.Stripper.startElement(Stripper.java:191)
at com.ddtek.xquery.saxon.event.ReceivingContentHandler.startElement(ReceivingContentHandler.java:279)
at com.ddtek.xquery.mediator.projection.ProjectingXMLReader.startElement(ProjectingXMLReader.java:328)
at com.ddtek.xmlconverter.platform.XWriter2SAX.element(XWriter2SAX.java:241)
at com.ddtek.xmlconverter.platform.XWriter2SAX.characters(XWriter2SAX.java:90)
at com.ddtek.xmlconverter.adapter.flat.Flat.parseFields(Flat.java:1007)
I tried running on server with below settings
-Xms1024m
-Xmx2048m
and got "java/lang/OutOfMemoryError"
My XQuery converts two csv files ( header and transaction) to xml and each header record is merged with transaction data and its written to an xml. ( see below code ).
Please advice how to run large data files
(:options:)
declare option ddtek:xml-streaming "yes";
declare option ddtek:serialize "encoding=UTF-8, omit-xml-declaration=no, indent=yes";
(:external variables:)
declare variable $esa-h-converter as xs:string external;
declare variable $esa-t-converter as xs:string external;
declare variable $input-h-data-path as xs:string external;
declare variable $input-t-data-path as xs:string external;
(:internal variables:)
(:functions:)
declare function local:substring-after-last($arg as xs:string?, $delim as xs:string) as xs:string
{
replace($arg, concat('^.*', local:escape-for-regex($delim)), '')
};
declare function local:escape-for-regex($arg as xs:string?) as xs:string
{
replace($arg, '(\.|\[|\]|\\|\||\-|\^|\$|\?|\*|\+|\{|\}|\(|\))', '\\$1')
};
(:main:)
<ROOT_NODE>
{
(:get all transaction records:)
let $transactions := doc(fn:concat("converter:", $esa-t-converter, "?file:///", $input-t-data-path))/DATA_ROOT/TRANSACTION
let $heads := doc(fn:concat("converter:", $esa-h-converter, "?file:///", $input-h-data-path))/DATA_ROOT/CUSTOMER[not(HD_CONT eq "111111111" and HD_CUSNAME eq "HEADER") and not(HD_CONT eq "999999999" and HD_CUSNAME eq "TRAILER")]
for $record at $primaryKey in $heads
let $header_id := concat(string($record/HD_CONT/text()), "_", string($record/HD_REGION/text()))
return
<ANZ_TEMPLATE>
<ESA_CUSTOMER>
<RECORD_COUNT>{ $primaryKey }</RECORD_COUNT>
<FILE_SOURCE>{ $input-h-data-path }</FILE_SOURCE>
<FILE_NAME>{ local:substring-after-last($input-h-data-path, "\") }</FILE_NAME>
<EFFDATE>{ $record/HD_TODATE/text() }</EFFDATE>
<JURISDICTION>ALL</JURISDICTION>
<LANGUAGE>English</LANGUAGE>
<BSP_RETURN_CODE></BSP_RETURN_CODE>
<BSP_DPID></BSP_DPID>
<BSP_DPID_BARCODE></BSP_DPID_BARCODE>
<BSP_SORT_PLAN></BSP_SORT_PLAN>
<BSP_WASHED_ADR1></BSP_WASHED_ADR1>
<BSP_WASHED_ADR2></BSP_WASHED_ADR2>
<BSP_WASHED_ADR3></BSP_WASHED_ADR3>
<BSP_WASHED_ADR4></BSP_WASHED_ADR4>
<BSP_WASHED_ADR5></BSP_WASHED_ADR5>
<BSP_WASHED_ADR6></BSP_WASHED_ADR6>
<HOPPER1></HOPPER1>
<HOPPER2></HOPPER2>
<HOPPER3></HOPPER3>
<HOPPER4></HOPPER4>
<HOPPER5></HOPPER5>
<HOPPER6></HOPPER6>
<UNIQUE_ID>{ $header_id }</UNIQUE_ID>
<HD_REGION>{ $record/HD_REGION/text() }</HD_REGION>
<HD_CONT>{ $record/HD_CONT/text() }</HD_CONT>
<HD_FRDATE>{ $record/HD_FRDATE/text() }</HD_FRDATE>
<HD_TODATE>{ $record/HD_TODATE/text() }</HD_TODATE>
<HD_CUSNAME>{ $record/HD_CUSNAME/text() }</HD_CUSNAME>
<HD_ADDR1>{ $record/HD_ADDR1/text() }</HD_ADDR1>
<HD_ADDR2>{ $record/HD_ADDR2/text() }</HD_ADDR2>
<HD_ADDR3>{ $record/HD_ADDR3/text() }</HD_ADDR3>
<HD_ADDR4>{ $record/HD_ADDR4/text() }</HD_ADDR4>
{
let $my-transactions := $transactions[concat(string(TR_CONT/text()), "_", string(TR_REGION/text())) = $header_id]
let $composed-rows := fn:distinct-values($my-transactions/fn:substring(TR_DATE/text(), 1, 4))
for $period in $composed-rows
return
<PERIOD>
<RECORD_COUNT>{ $primaryKey }</RECORD_COUNT>
<YEAR>{ $period }</YEAR>
{
for $local-transactions in $my-transactions[fn:starts-with(TR_DATE, $period)]
return
<TRANSACTION>
<RECORD_COUNT>{ $primaryKey }</RECORD_COUNT>
<TR_REGION>{ $local-transactions/TR_REGION/text() }</TR_REGION>
<TR_CONT>{ $local-transactions/TR_CONT/text() }</TR_CONT>
<TR_DATE>{ $local-transactions/TR_DATE/text() }</TR_DATE>
<TR_TEXT>{ $local-transactions/TR_TEXT/text() }</TR_TEXT>
{
if(fn:number($local-transactions/TR_DEBIT/text()) eq 0) then ()
else (<TR_DEBIT>{ $local-transactions/TR_DEBIT/text() }</TR_DEBIT>)
}
{
if(fn:number($local-transactions/TR_CREDIT/text()) eq 0) then ()
else (<TR_CREDIT>{ $local-transactions/TR_CREDIT/text() }</TR_CREDIT>)
}
<TR_BALANCE>{ $local-transactions/TR_BALANCE/text() }</TR_BALANCE>
<TR_AMT1>{ $local-transactions/TR_AMT1/text() }</TR_AMT1>
</TRANSACTION>
}
</PERIOD>
}
</ESA_CUSTOMER>
</ANZ_TEMPLATE>
}
</ROOT_NODE>
|
|
|
|
top