0
에폭에서 셔플의 문장과 문제는 word2vec 및 doc2vec이 우수한 자습서, here 및 here을 사용하고 코드 샘플을 사용하려고 시작합니다. 나는 line_clean() 구두점, 불어 등을 제거하는 방법을 추가했습니다.이 doc2vec/gensim - 내가 노력하고
그러나 훈련 반복에서 호출 된 line_clean() 메서드에 문제가 있습니다. 전 세계적인 방법에 대한 호출이 그것을 망쳐 놓는 것을 이해합니다. 그러나이 문제를 피하는 방법을 모르겠습니다.
Iteration 1
Traceback (most recent call last):
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 96, in <module>
train()
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 91, in train
model.train(sentences.sentences_perm(),total_examples=model.corpus_count,epochs=model.iter)
File "/Users/santino/Dev/doc2vec_exp/doc2vec_exp_app/doc2vec/untitled.py", line 61, in sentences_perm
shuffled = list(self.sentences)
AttributeError: 'TaggedLineSentence' object has no attribute 'sentences'
내 코드는 다음과 같습니다 :
import gensim
from gensim import utils
from gensim.models.doc2vec import TaggedDocument
from gensim.models import Doc2Vec
import os
import random
import numpy
from sklearn.linear_model import LogisticRegression
import logging
import sys
from nltk import RegexpTokenizer
from nltk.corpus import stopwords
tokenizer = RegexpTokenizer(r'\w+')
stopword_set = set(stopwords.words('english'))
def clean_line(line):
new_str = unicode(line, errors='replace').lower() #encoding issues
dlist = tokenizer.tokenize(new_str)
dlist = list(set(dlist).difference(stopword_set))
new_line = ' '.join(dlist)
return new_line
class TaggedLineSentence(object):
def __init__(self, sources):
self.sources = sources
flipped = {}
# make sure that keys are unique
for key, value in sources.items():
if value not in flipped:
flipped[value] = [key]
else:
raise Exception('Non-unique prefix encountered')
def __iter__(self):
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
yield TaggedDocument(utils.to_unicode(clean_line(line)).split(), [prefix + '_%s' % item_no])
def to_array(self):
self.sentences = []
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
self.sentences.append(TaggedDocument(utils.to_unicode(clean_line(line)).split(), [prefix + '_%s' % item_no]))
return(self.sentences)
def sentences_perm(self):
shuffled = list(self.sentences)
random.shuffle(shuffled)
return(shuffled)
def train():
#create a list data that stores the content of all text files in order of their names in docLabels
doc_files = [f for f in os.listdir('./data/') if f.endswith('.csv')]
sources = {}
for doc in doc_files:
doc2 = os.path.join('./data',doc)
sources[doc2] = doc.replace('.csv','')
sentences = TaggedLineSentence(sources)
# #iterator returned over all documents
model = gensim.models.Doc2Vec(size=300, min_count=2, alpha=0.025, min_alpha=0.025)
model.build_vocab(sentences)
#training of model
for epoch in range(10):
#random.shuffle(sentences)
print 'iteration '+str(epoch+1)
#model.train(it)
model.alpha -= 0.002
model.min_alpha = model.alpha
model.train(sentences.sentences_perm(),total_examples=model.corpus_count,epochs=model.iter)
#saving the created model
model.save('reddit.doc2vec')
print "model saved"
train()