#!python
# coding: utf-8
# --------------------------------------------
# Make data for test.
with file('g:\\data.txt', 'w') as fd:
for i in range(1000):
fd.write('%d\n' % i)
fd.close()
# --------------------------------------------
# Use generator for batch processing
def get_data(filename, num_of_data):
"""Gather data for batch processing."""
data_collection = []
for i,ln in enumerate(file(filename, 'r'), 1):
data_collection.append(ln.strip())
if i % num_of_data == 0:
yield data_collection
data_collection = []
for data_50 in get_data('g:\\data.txt', 50):
# process data here.
data_sum = sum(map(int, data_50))
print data_sum
# --------------------------------------------
# Use cmdlet package to handle same task.
from cmdlet.cmds import *
for data_50 in (readline('g:\\data.txt') | pack(50)):
# process data here.
data_sum = sum(map(int, data_50))
print data_sum
