ãã®èšäºã¯ã ãããã£ã°ã«ãŒã Advent Calendar 2022 11æ¥ç®ã®èšäºã§ãã ã¯ããã« äŒå¡ã·ã¹ãã ã°ã«ãŒã N1!Machine Learning Product Engineerã®äžæã§ãã ãããã£ã§ã¯11/22ã«NIFTY Tech Dayãšããã€ãã³ããéå¬ããŸããã ãã®æã«ããããã£ãã¥ãŒã¹ã«ãããæ·±å±€èªç¶èšèªåŠçã«ãããã¥ãŒã¹èšäºèŠçŽã«ã€ããŠçºè¡šããããŠããã ããŸãããããã®æã«è¡ã£ã転移åŠç¿ïŒãã¡ã€ã³ãã¥ãŒãã³ã°ïŒã«ã€ããŠæè¡çãªè§£èª¬ãããŠãããŸãã ã¢ãã«ã®è»¢ç§»åŠç¿ã«ã€ã㊠æ¬èšäºã®å€§ãŸããªå®è£
㯠sonoisa ããã®èšäºãåèã«æ§ç¯ããŠããŸãã https://qiita.com/sonoisa/items/a9af64ff641f0bbfed44 èšèªã¢ãã«ã®è»¢ç§»åŠç¿ çŸåšã®èšèªã¢ãã«ã¯éåžžã«å€§èŠæš¡ã§ãããå
šãåŠç¿ãããŠããªãç¶æ
ïŒã¹ã¯ã©ããïŒããæ¹ããŠåŠç¿ãããããšã¯çŸå®çã§ã¯ãããŸãããããã§èæ¡ãããã®ã転移åŠç¿ãšããææ³ã§ãã 転移åŠç¿ãšã¯ãæ¢ã«åŠç¿ãããå€§èŠæš¡ãªèšèªã¢ãã«ãå
ã«ãæ°ããã¿ã¹ã¯ã«ã€ããŠã®åŠç¿ãè¡ãææ³ã§ãããã®æ¹æ³ã¯ãæ°ããã¿ã¹ã¯ã®åŠç¿ããŒã¿ãå°ãªãå Žåã«ç¹ã«æå¹ã§ãããŸããæ¢ã«åŠç¿ãããã¢ãã«ã®åŠç¿æžã¿ã®ç¥èãåŒãç¶ãããšã§ãåŠç¿ãã¹ã ãŒãºã«é²ããããåŠç¿æéã®ççž®ãæåŸ
ãããŸãã ããã³ããåŠç¿ https://arxiv.org/abs/1910.10683 ä»åã®ãã¥ãŒã¹èšäºèŠçŽã§ã¯T5ãšããã¢ãã«ã®åŠç¿ãè¡ããŸããããã®åŠç¿ã«ã¯ããã³ãããçšããŠåŠç¿ãè¡ããŸããçŸåšã®æ·±å±€åŠç¿ã¢ãã«ã¯éåžžã«å€§èŠæš¡ã§ããã1ã€ã®ã¢ãã«ã§è€æ°ã®ã¿ã¹ã¯ãå®è¡å¯èœã§ããããã³ãããšåŒã°ããæ¥é èŸãé ã«ä»ããŠåŠç¿ãè¡ãããšã§ãæç« ã«å¯ŸããŠäœãè¡ãããããæå®ããããã«åãããã¿ã¹ã¯ãå®è¡ããããã«åŠç¿ããŸãã å®è£
ãšåŠç¿ã®å®è¡ ããã§ã¯ãããã£ãã¥ãŒã¹ã«ãããèšäºããŒã¿ãšèŠçŽããŒã¿ãçšããŠãT5ã¢ãã«ã®è»¢ç§»åŠç¿ãè¡ããŸãã ããŒã¿ã»ããã®æºå ããŒãã©ã€ãºåŠç ãŸããneologdã®æ£èŠååŠçãæ¹å€ããåŠçãçšããŠãæ£èŠååŠçãå®çŸ©ããŸã # https://github.com/neologd/mecab-ipadic-neologd/wiki/Regexp.ja ããåŒçšã»äžéšæ¹å€ from __future__ import unicode_literals import re import unicodedata def unicode_normalize(cls, s): pt = re.compile('([{}]+)'.format(cls)) def norm(c): return unicodedata.normalize('NFKC', c) if pt.match(c) else c s = ''.join(norm(x) for x in re.split(pt, s)) s = re.sub('ïŒ', '-', s) return s def remove_extra_spaces(s): s = re.sub('[ ã]+', ' ', s) blocks = ''.join(('\u4E00-\u9FFF', # CJK UNIFIED IDEOGRAPHS '\u3040-\u309F', # HIRAGANA '\u30A0-\u30FF', # KATAKANA '\u3000-\u303F', # CJK SYMBOLS AND PUNCTUATION '\uFF00-\uFFEF' # HALFWIDTH AND FULLWIDTH FORMS )) basic_latin = '\u0000-\u007F' def remove_space_between(cls1, cls2, s): p = re.compile('([{}]) ([{}])'.format(cls1, cls2)) while p.search(s): s = p.sub(r'\1\2', s) return s s = remove_space_between(blocks, blocks, s) s = remove_space_between(blocks, basic_latin, s) s = remove_space_between(basic_latin, blocks, s) return s def normalize_neologd(s): s = s.strip() s = unicode_normalize('ïŒ-ïŒïŒ¡-ïœ-ïœïœ¡-ïŸ', s) def maketrans(f, t): return {ord(x): ord(y) for x, y in zip(f, t)} s = re.sub('[ËÖââââââ»ââ]+', '-', s) # normalize hyphens s = re.sub('[ï¹£ïŒïœ°ââââãŒ]+', 'ãŒ', s) # normalize choonpus s = re.sub('[~âŒâŸã ïœ]+', 'ã', s) # normalize tildes (modified by Isao Sonobe) s = s.translate( maketrans('!"#$%&\'()*+,-./:;<=>?@[Â¥]^_`{|}~', 'ïŒâïŒïŒïŒ
ïŒâïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒ ¥ïœïœïœïœãããã»ãã')) s = remove_extra_spaces(s) s = unicode_normalize('ïŒâïŒïŒïŒ
ïŒâïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒïŒ ¥ïœïœïœïœã', s) # keep ïŒ,ã»,ã,ã s = re.sub('[â]', '\'', s) s = re.sub('[â]', '"', s) return s ãã®ã»ãã«ã¿ãæåãã¿ã°ã®é€å»ããšã¹ã±ãŒãã·ãŒã±ã³ã¹ã®åŸ©å·åãªã©ãè¡ããããŒã¿ã»ãããæŽåããŸãã åèå
ã®åŠçã§ã¯loweråŠçãã¹ããŒã¹ã®åé€ãªã©ãå«ãŸããŠããŸãããäŸãã°ãiPhoneããªã©ã®åååãã空çœã«ãã£ãŠæå³çã«åºåãããŠããéšåãæ¶æ»
ãããªã©ã®çŸè±¡ãèµ·ãã£ããããæ¬èšäºã§ã¯ãã®éšåã®åŠçãè¡ããªãããã«å®è£
ããŠããŸãã ãããã£ãã¥ãŒã¹ã§ã¯çæãšé·æã®2çš®é¡ã®ã¿ã€ãã«ã®ä»ã3è¡èŠçŽãäœæããŠããããã®ããŒã¿ã«ããããããã³ãããä»äžãåŠç¿ãããŸãã import re import numpy as np import pickle from tqdm import tqdm tag_regex = re.compile(r"<[^>]*?>") def normalize_text(text): text = text.replace("\t", " ") text = normalize_neologd(text) text = tag_regex.sub("", text) text = text.replace("&quot;", "\"").replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">").replace("&nbsp;", " ") return text all_data = [] count = 0 for index, data in news_data.iterrows(): if data['body'] is None or data['body'] is np.nan or not data['body']: continue normalized_body = normalize_text(data['body']) all_data.append({"text": "keyword: " + normalized_body,"response": normalize_text(data['keyword_str_1']),}) all_data.append({"text": "keyword: " + normalized_body,"response": normalize_text(data['keyword_str_2']),}) all_data.append({"text": "keyword: " + normalized_body,"response": normalize_text(data['keyword_str_3']),}) all_data.append({"text": "topics_title: " + normalized_body,"response": normalize_text(data['topics_article_title']),}) all_data.append({"text": "title: " + normalized_body,"response": normalize_text(data['title']),}) if data['long_title'] is not None and data['long_title'] is not np.nan: all_data.append({"text": "long_title: " + normalized_body,"response": normalize_text(data['long_title']),}) if data['summary_1'] is not None and data['summary_1'] is not np.nan: all_data.append({"text": "summary_1: " + normalized_body,"response": normalize_text(data['summary_1']),}) if data['summary_2'] is not None and data['summary_2'] is not np.nan: all_data.append({"text": "summary_2: " + normalized_body,"response": normalize_text(data['summary_2']),}) if data['summary_3'] is not None and data['summary_3'] is not np.nan: all_data.append({"text": "summary_3: " + normalized_body,"response": normalize_text(data['summary_3']),}) ããã³ãããå«ããæ¬æã§ããtextãšãããã«å¯Ÿå¿ããå¿çã§ããresponseããall_dataãšããé
åã«å«ããç¶æ
ã«ãªããŸããã ããŒã¿ã»ããã®åå² ããŒã¿ãtrain/validation/testã®3ã€ã«åå²ããŸãã import random from tqdm import tqdm random.seed(1234) random.shuffle(all_data) def to_line(data): text = data["text"] response = data["response"] assert len(text) > 0 and len(response) > 0 return f"{text}\t{response}\n" data_size = len(all_data) train_ratio, val_ratio, test_ratio = 0.95, 0.03, 0.02 with open(f"data/train.tsv", "w", encoding="utf-8") as f_train, \ open(f"data/val.tsv", "w", encoding="utf-8") as f_val, \ open(f"data/test.tsv", "w", encoding="utf-8") as f_test: for i, data in tqdm(enumerate(all_data)): line = to_line(data) if i < train_ratio * data_size: f_train.write(line) elif i < (train_ratio + val_ratio) * data_size: f_val.write(line) else: f_test.write(line) 確èªããŠã¿ããšãã©ã³ãã ã«ããŒã¿ãåå²ãããããšãããããŸãã åŠç¿ã®å®è¡ ã¢ãã«ã®å®çŸ©ãªã©ã¯åèå
ãšåããããããã§ã¯å²æããŸãã https://qiita.com/sonoisa/items/a9af64ff641f0bbfed44 äºååŠç¿ã¢ãã«ã«ã¯ megagonlabs/t5-base-japanese-web ã䜿çšããŸãã A100(80GB)æã®èšå®ã«ã€ã㊠Google Colaboratoryãªã©ã§GPUã«A100ã䜿ãå Žåãããªã€ã³ã¹ããŒã«ãããŠããtorchã§ã¯CUDAã察å¿ããŠããªããšãããšã©ãŒãèµ·ããŸãããã®å Žåã«ã¯ã以äžã®ã³ãã³ãã§å¯Ÿå¿ããtorchãªã©ãã€ã³ã¹ããŒã«ããŸãã !pip install -qU transformers[ja] pytorch_lightning sentencepiece torch==1.10.0+cu111 torchvision==0.11.1+cu111 torchaudio torchtext -f https://download.pytorch.org/whl/torch_stable.html ãã®ä»ãã€ããŒãã©ã¡ãŒã¿ ãã€ããŒãã©ã¡ãŒã¿ã«ã€ããŠã¯ä»¥äžã®ããã«èšå®ããŸãã ã¢ãã«ã®ãã§ãã¯ãã€ã³ãåšãã®å®çŸ©ãè¡ã£ãŠãããšããããéäžã§åŠç¿ãçµäºããïŒPCãæ¢ãŸããGoogle Colaboratoryã®ã»ãã·ã§ã³ãåããŠããŸãïŒèªäœã«é¥ã£ãŠããåŠç¿ãéäžããå§ããããšãå¯èœãªããèšå®ããŠããããšãããããããŸãã ïŒã©ãããŠãæéãããã£ãŠããŸããããªããã®ãããªå€§èŠæš¡ã¢ãã«ã®åŠç¿ã§ã¯éåžžã«äŸ¿å©ã ãšæããŸããïŒ # åŠç¿ã«çšãããã€ããŒãã©ã¡ãŒã¿ãèšå®ãã args_dict.update({ "max_input_length": 1024, # å
¥åæã®æå€§ããŒã¯ã³æ° "max_target_length": 64, # åºåæã®æå€§ããŒã¯ã³æ° "train_batch_size": 8, "eval_batch_size": 8, "num_train_epochs": 2, }) args = argparse.Namespace(**args_dict) train_params = dict( accumulate_grad_batches=args.gradient_accumulation_steps, gpus=args.n_gpu, max_epochs=args.num_train_epochs, precision= 16 if args.fp_16 else 32, amp_backend='apex', amp_level=args.opt_level, gradient_clip_val=args.max_grad_norm, default_root_dir=f"{MODEL_SAVE_DIR}/checkpoint", ) 以äžãå®è¡ããŠã転移åŠç¿ãè¡ããŸãã # 転移åŠç¿ã®å®è¡ model = T5FineTuner(args) trainer = pl.Trainer(**train_params) trainer.fit(model) # æçµãšããã¯ã®ã¢ãã«ãä¿å model.tokenizer.save_pretrained(MODEL_DIR) model.model.save_pretrained(MODEL_DIR) æšè«åŠç 以äžã®ã³ãŒããå®è¡ããããšã§æšè«åŠçãè¡ããŸãã article_body = "æ¬æ" MAX_SOURCE_LENGTH = 1024 # å
¥åãããèšäºæ¬æã®æå€§ããŒã¯ã³æ° MAX_TARGET_LENGTH = 64 # çæãããåºåã®æå€§ããŒã¯ã³æ° import re import pickle from tqdm import tqdm tag_regex = re.compile(r"<[^>]*?>") def normalize_text(text): text = text.replace("\t", " ") text = normalize_neologd(text) text = tag_regex.sub("", text) return text def preprocess_body(text): return normalize_text(text.replace("\n", " ")) # æšè«ã¢ãŒãèšå® trained_model.eval() # ååŠçãšããŒã¯ãã€ãºãè¡ã preprocessed_body = preprocess_body(article_body) inputs = ["title: " + preprocessed_body, "long_title: " + preprocessed_body, "topics_title: " + preprocessed_body, "summary_1: " + preprocessed_body, "summary_2: " + preprocessed_body, "summary_3: " + preprocessed_body] batch = tokenizer.batch_encode_plus( inputs, max_length=MAX_SOURCE_LENGTH, truncation=True, padding="longest", return_tensors="pt") input_ids = batch['input_ids'] input_mask = batch['attention_mask'] if USE_GPU: input_ids = input_ids.cuda() input_mask = input_mask.cuda() # çæåŠçãè¡ã outputs = trained_model.generate( input_ids=input_ids, attention_mask=input_mask, max_length=MAX_TARGET_LENGTH, temperature=1.0, # çæã«ã©ã³ãã æ§ãå
¥ããæž©åºŠãã©ã¡ãŒã¿ num_beams=10, # ããŒã ãµãŒãã®æ¢çŽ¢å¹
diversity_penalty=1.0, # çæçµæã®å€æ§æ§ãçã¿åºãããã®ããã«ã㣠num_beam_groups=10, # ããŒã ãµãŒãã®ã°ã«ãŒãæ° num_return_sequences=1, # çæããæã®æ° repetition_penalty=1.5, # åãæã®ç¹°ãè¿ãïŒã¢ãŒã厩å£ïŒãžã®ããã«ã㣠) # çæãããããŒã¯ã³åãæååã«å€æãã generated = [tokenizer.decode(ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) for ids in outputs] # çæãããæååã衚瀺ãã for i, generated_str in enumerate(generated): if i == 0: print(f"title: {generated_str}") elif i == 1: print(f"long_title: {generated_str}") elif i == 2: print(f"topics_title: {generated_str}") else: print(f"summary {i-2}: {generated_str}") # ååŠçãšããŒã¯ãã€ãºãè¡ã ã®éšåã§ããã³ãããäžãã€ã€ãããŒã¯ãã€ãºãè¡ããŸãã # çæåŠçãè¡ã ã®çæåŠçã§ã¯ãã€ããŒãã©ã¡ãŒã¿ã倿Žããããšã§ãåŠçæéã粟床ã®ãã¬ãŒããªãé¢ä¿ã調æŽããŸãã å®éã®æšè« å®éã®æšè«ã¯ä»¥äžã®ããã«ãªããŸããïŒNIFTY Tech Day 2022ããïŒ å
šãŠã«ãããŠæåããããã§ã¯ãªãã以äžã®ããã«å€±æããäŸããããŸãã ãããã« ä»åã¯NIFTY Tech Day 2022ã§ã¯è©±ããªãã£ã现ãã転移åŠç¿ã®ææ³ãªã©ã«ã€ããŠæžããŠããŸããã æ·±å±€åŠç¿åšãã®èªç¶èšèªã®æŽå²ããã¯ã©ãŠãäžã®ã¢ãŒããã¯ãã£ã«é¢ããŠãNIFTY Tech Day 2022ã§ã話ããã®ã§ã èå³ãããæ¹ã¯ãã²åŸ¡èЧãã ããã æè¿ã ãšChatGPTã®ãããªæµæ¢ãªå¯Ÿè©±åã®AIæè¡ãç»å Žããããããèªç¶èšèªåŠçã¯äººéã«è¿ãååšã«ãªãã€ã€ãããŸããã å®éã«æ·±å±€èªç¶èšèªåŠçã«ããèŠçŽçæãªã©ã«ææŠããŠã©ã®ãããªèŠåŽããã£ãã®ãïŒã«ã€ããŠããŸãã©ããã®æ©äŒã«çºè¡šããããšæããŸãã We are hiring! ãããã£ã§ã¯ãããŸããŸãªãããã¯ããžææŠãããšã³ãžãã¢ãçµ¶è³åéäžã§ãïŒ ãèå³ã®ããæ¹ã¯ä»¥äžã®æ¡çšãµã€ããããæ°è»œã«ãé£çµ¡ãã ããïŒ ãããã£æ ªåŒäŒç€Ÿæ¡çšæ
å ± Tech TalkãMeetUpãéå¬ããŠãããŸãïŒ ãã¡ãããæ°è»œã«ãå¿åãã ããïŒ Event â NIFTY engineering ãã®èšäºã¯ã ãããã£ã°ã«ãŒã Advent Calendar 2022 11æ¥ç®ã®èšäºã§ãã ææ¥ã¯ ãããã£ã©ã€ãã¹ã¿ã€ã« ã®saikeiããã§ããæ¬¡åããæ¥œãã¿ã«ïŒ