type EachToken
stream::IOStream
EachToken(filename::String) = new(open(filename, "r"))
end
start(itr::EachToken) = nada
function next(itr::EachToken, nada)
token = readutil(itr.stream, " ")
return token[1: end - 1]
end
done(itr::EachToken, nada) = eof(itr.stream)
function counted_tokens(token_iterator)
counts = Dict{String, Int32}()
for token=token_iterator
counts[token] = get(counts, token, 0) + 1
end
return counts
end
token_it = EachToken("small_text.txt")
counts = counted_tokens(token_it)
println(counts)
# compiler output
# ERROR: no method start(EachToken,)
# in counted_tokens at /home/rrenaud/word2vec_jl/word_to_vec.jl:16
# in include at boot.jl:238
# at /home/rrenaud/word2vec_jl/word_to_vec.jl:24
