a h @sldZddlmZmZmZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZdd lmZmZmZmZmZmZdd lmZmZdd lmZmZmZdd lmZm Z m!Z!d dl"m#Z#e!$e%Z&ej'Z(Gdddej)Z*d;ej)ej+ej+ej+eej+e,e,eej+dddZ-Gdddej)Z.Gdddej)Z/Gdddej)Z0Gdddej)Z1Gdddej)Z2Gdd d eZ3Gd!d"d"ej)Z4Gd#d$d$ej)Z5Gd%d&d&ej)Z6Gd'd(d(ej)Z7Gd)d*d*ej)Z8eGd+d,d,eZ9eGd-d.d.e9Z:eGd/d0d0e9Z;ed1d2Gd3d4d4e9Zgd:Z?dS)t|ds>td|jd|jd||_|j|_t|j|j|_|j|j|_ t |j|j |_ t |j|j |_ t |j|j |_t |j|_|j|_|jd|_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g)r#r$r'num_attention_headshasattr ValueErrorr9intattention_head_size all_head_sizerLinearrUrVrWr2Zattention_probs_dropout_probr4attention_dropoutrYr7r:r<r=r$s"  zLayoutLMSelfAttention.__init__NF hidden_statesrXrZoutput_attentionsreturncKs|jdd}g|d|jR}|||dd}|||dd} |||dd} t} |jj dkrt |jj } | ||| | |f|j sdn|j |j |d|\} } | jg|dR} |r| | fn| f}|S)Nr!rrAeagerrS)r4rYrZ)rarorUrcr`rVrWrhr9Z_attn_implementationrr^rrrYZreshaperd)r8rtrXrZrurerJZ hidden_shapeZ query_statesZ key_statesZ value_statesZattention_interfacergrfoutputsr<r<r=rMs0    zLayoutLMSelfAttention.forward)NNF) rNrOrPr$r5Tensorr FloatTensorbooltuplerMrRr<r<r:r=ris rics4eZdZfddZejejejdddZZS)LayoutLMSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr) r#r$rrqr'denser1r0r2r3r4r7r:r<r=r$s zLayoutLMSelfOutput.__init__rt input_tensorrvcCs&||}||}|||}|SNrr4r1r8rtrr<r<r=rMs  zLayoutLMSelfOutput.forwardrNrOrPr$r5ryrMrRr<r<r:r=r}s r}csTeZdZfddZddZd ejeejeejee e ejddd Z Z S) LayoutLMAttentioncs*tt||_t||_t|_dSr)r#r$rir8r}outputset pruned_headsr7r:r<r=r$s   zLayoutLMAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr\)lenrr8rkrorrrUrVrWrrrpunion)r8headsindexr<r<r= prune_headss zLayoutLMAttention.prune_headsNFrsc Ks@|j|f|||d|}||d|}|f|dd}|SN)rXrZrurr)r8r) r8rtrXrZrureZ self_outputsattention_outputrxr<r<r=rMszLayoutLMAttention.forward)NNF) rNrOrPr$rr5ryrrzr{r|rMrRr<r<r:r=rs rcs0eZdZfddZejejdddZZS)LayoutLMIntermediatecsBtt|j|j|_t|jt r6t |j|_ n|j|_ dSr) r#r$rrqr'intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr7r:r<r=r$s   zLayoutLMIntermediate.__init__rtrvcCs||}||}|Sr)rrr8rtr<r<r=rMs  zLayoutLMIntermediate.forwardrr<r<r:r=rs rcs4eZdZfddZejejejdddZZS)LayoutLMOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr~) r#r$rrqrr'rr1r0r2r3r4r7r:r<r=r$ s zLayoutLMOutput.__init__rcCs&||}||}|||}|Srrrr<r<r=rM&s  zLayoutLMOutput.forwardrr<r<r:r=rs rcsTeZdZfddZd ejeejeejeee ejdddZ dd Z Z S) LayoutLMLayercs:t|j|_d|_t||_t||_t||_ dS)Nr) r#r$chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr7r:r<r=r$/s    zLayoutLMLayer.__init__NFrsc KsP|j|f|||d|}|d}|dd}t|j|j|j|} | f|}|Sr)rrfeed_forward_chunkrr) r8rtrXrZrureZself_attention_outputsrrx layer_outputr<r<r=rM7s   zLayoutLMLayer.forwardcCs||}|||}|Sr)rr)r8rZintermediate_outputrr<r<r=rPs  z LayoutLMLayer.feed_forward_chunk)NNF) rNrOrPr$r5ryrrzr{r|rMrrRr<r<r:r=r.s  rc sdeZdZfddZed ejeejeejee ee ee e e eje fdddZ ZS) LayoutLMEncodercs:t|_tfddtjD|_d|_dS)Ncsg|] }tqSr<)r).0ir9r<r= [z,LayoutLMEncoder.__init__..F) r#r$r9rZ ModuleListrangenum_hidden_layerslayerZgradient_checkpointingr7r:rr=r$Xs  zLayoutLMEncoder.__init__NFT)rtrXrZruoutput_hidden_states return_dictrvcKs|rdnd}|rdnd} t|jD]\\} } |r8||f}|durH|| nd} | f||| |d|} | d}|r"| | df} q"|r||f}t||| dS)Nr<)rtrXrZrurr)last_hidden_statert attentions) enumeraterr )r8rtrXrZrurrreZall_hidden_statesZall_self_attentionsrZ layer_moduleZlayer_head_maskZ layer_outputsr<r<r=rM^s0    zLayoutLMEncoder.forward)NNFFT)rNrOrPr$rr5ryrrzr{rr|r rMrRr<r<r:r=rWs  rcs0eZdZfddZejejdddZZS)LayoutLMPoolercs*tt|j|j|_t|_dSr)r#r$rrqr'rZTanh activationr7r:r<r=r$s zLayoutLMPooler.__init__rcCs(|dddf}||}||}|S)Nr)rr)r8rtZfirst_token_tensor pooled_outputr<r<r=rMs  zLayoutLMPooler.forwardrr<r<r:r=rs rcs0eZdZfddZejejdddZZS)LayoutLMPredictionHeadTransformcsVtt|j|j|_t|jtr6t |j|_ n|j|_ tj |j|j d|_ dSr~) r#r$rrqr'rrrrr transform_act_fnr1r0r7r:r<r=r$s   z(LayoutLMPredictionHeadTransform.__init__rcCs"||}||}||}|Sr)rrr1rr<r<r=rMs   z'LayoutLMPredictionHeadTransform.forwardrr<r<r:r=rs rcs,eZdZfddZddZddZZS)LayoutLMLMPredictionHeadcsLtt||_tj|j|jdd|_t t |j|_ |j |j_ dS)NF)bias) r#r$r transformrrqr'r&decoder Parameterr5rCrr7r:r<r=r$s   z!LayoutLMLMPredictionHead.__init__cCs|j|j_dSr)rrr8r<r<r= _tie_weightssz%LayoutLMLMPredictionHead._tie_weightscCs||}||}|Sr)rrrr<r<r=rMs  z LayoutLMLMPredictionHead.forward)rNrOrPr$rrMrRr<r<r:r=rs rcs0eZdZfddZejejdddZZS)LayoutLMOnlyMLMHeadcstt||_dSr)r#r$r predictionsr7r:r<r=r$s zLayoutLMOnlyMLMHead.__init__)sequence_outputrvcCs||}|Sr)r)r8rprediction_scoresr<r<r=rMs zLayoutLMOnlyMLMHead.forwardrr<r<r:r=rs rc@s&eZdZUeed<dZdZddZdS)LayoutLMPreTrainedModelr9layoutlmTcCst|tjr:|jjjd|jjd|jdur|jj n|t|tj rz|jjjd|jjd|j dur|jj|j  n>> from transformers import AutoTokenizer, LayoutLMModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> outputs = model( ... input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids ... ) >>> last_hidden_states = outputs.last_hidden_state ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer!z5You have to specify either input_ids or inputs_embeds)r@r>)rrA)r?rr)rFrGr rHrIT)rZrurr)rZ pooler_outputrtr)r9ruruse_return_dictrmZ%warn_if_padding_and_no_attention_maskrBr@r5ZonesrCrDZ unsqueezerbr?Zfinfominr\r6rnext parametersrLrrr rtr)r8rFrGrXrHr rZrIrurrrJr@Zextended_attention_maskZembedding_outputZencoder_outputsrrr<r<r=rMsj5     zLayoutLMModel.forward) NNNNNNNNNN)rNrOrPr$rrrrrrr5 LongTensorrzr{rr|r rMrRr<r<r:r=rs:  rcseZdZddgZfddZddZddZd d Zee de e j e e j e e j e e j e e j e e j e e j e e j e ee ee eeeefd d dZZS)LayoutLMForMaskedLMzcls.predictions.decoder.biaszcls.predictions.decoder.weightcs,t|t||_t||_|dSr)r#r$rrrclsrr7r:r<r=r$}s   zLayoutLMForMaskedLM.__init__cCs |jjjSrrrLr(rr<r<r=rsz(LayoutLMForMaskedLM.get_input_embeddingscCs |jjjSr)rrrrr<r<r=get_output_embeddingssz)LayoutLMForMaskedLM.get_output_embeddingscCs||jj_|j|jj_dSr)rrrr)r8Znew_embeddingsr<r<r=set_output_embeddingss z)LayoutLMForMaskedLM.set_output_embeddingsN rFrGrXrHr rZrIlabelsrurrrvc  Cs| dur | n|jj} |j|||||||| | dd } | d} || }d}|durrt}||d|jj|d}t||| j| j dS)a2 bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMForMaskedLM >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForMaskedLM.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "[MASK]"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> labels = tokenizer("Hello world", return_tensors="pt")["input_ids"] >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=labels, ... ) >>> loss = outputs.loss ```NT)rXrHr rZrIrurrrr!losslogitsrtr) r9rrrrrcr&rrtr)r8rFrGrXrHr rZrIrrurrrxrrZmasked_lm_lossloss_fctr<r<r=rMs8@  zLayoutLMForMaskedLM.forward) NNNNNNNNNNN)rNrOrPZ_tied_weights_keysr$rrrrrrr5rrzr{rr|rrMrRr<r<r:r=rys@  rz LayoutLM Model with a sequence classification head on top (a linear layer on top of the pooled output) e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. )Z custom_introcseZdZfddZddZeed eej eej eej eej eej eej eej eej ee ee ee e e efd ddZZS) !LayoutLMForSequenceClassificationcsJt||j|_t||_t|j|_t |j |j|_ | dSr r#r$ num_labelsrrrr2r3r4rqr' classifierrr7r:r<r=r$s   z*LayoutLMForSequenceClassification.__init__cCs |jjjSrrrr<r<r=rsz6LayoutLMForSequenceClassification.get_input_embeddingsNrc  CsN| dur | n|jj} |j|||||||| | dd } | d} || } || }d}|dur:|jjdur|jdkr|d|j_n4|jdkr|jtj ks|jtj krd|j_nd|j_|jjdkrt }|jdkr|| | }n |||}nN|jjdkrt }||d|j|d}n|jjdkr:t}|||}t||| j| jd S) aB bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMForSequenceClassification >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> sequence_label = torch.tensor([1]) >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=sequence_label, ... ) >>> loss = outputs.loss >>> logits = outputs.logits ```NT rFrGrXrHr rZrIrurrrZ regressionZsingle_label_classificationZmulti_label_classificationr!r)r9rrr4rZ problem_typerr?r5rDrnrsqueezerrcrrrtr)r8rFrGrXrHr rZrIrrurrrxrrrrr<r<r=rMsR@       "     z)LayoutLMForSequenceClassification.forward) NNNNNNNNNNN)rNrOrPr$rrrrr5rrzr{rr|rrMrRr<r<r:r=rs:  ra3 LayoutLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for sequence labeling (information extraction) tasks such as the [FUNSD](https://guillaumejaume.github.io/FUNSD/) dataset and the [SROIE](https://rrc.cvc.uab.es/?ch=13) dataset. cseZdZfddZddZeed eej eej eej eej eej eej eej eej ee ee ee e e efd ddZZS) LayoutLMForTokenClassificationcsJt||j|_t||_t|j|_t |j |j|_ | dSrrr7r:r<r=r$s   z'LayoutLMForTokenClassification.__init__cCs |jjjSrrrr<r<r=rsz3LayoutLMForTokenClassification.get_input_embeddingsNrc  Cs| dur | n|jj} |j|||||||| | dd } | d} || } || }d}|durzt}||d|j|d}t||| j | j dS)a bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMForTokenClassification >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> token_labels = torch.tensor([1, 1, 0, 0]).unsqueeze(0) # batch size of 1 >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=token_labels, ... ) >>> loss = outputs.loss >>> logits = outputs.logits ```NTrrr!r) r9rrr4rrrcrrrtr)r8rFrGrXrHr rZrIrrurrrxrrrrr<r<r=rMs4>   z&LayoutLMForTokenClassification.forward) NNNNNNNNNNN)rNrOrPr$rrrrr5rrzr{rr|rrMrRr<r<r:r=rys:  rcseZdZd fdd ZddZeed eej eej eej eej eej eej eej eej eej ee ee ee e e efd dd ZZS) LayoutLMForQuestionAnsweringTcs<t||j|_t||_t|j|j|_| dS)z has_visual_segment_embedding (`bool`, *optional*, defaults to `True`): Whether or not to add visual segment embeddings. N) r#r$rrrrrqr' qa_outputsr)r8r9Zhas_visual_segment_embeddingr:r<r=r$s   z%LayoutLMForQuestionAnswering.__init__cCs |jjjSrrrr<r<r=rsz1LayoutLMForQuestionAnswering.get_input_embeddingsN) rFrGrXrHr rZrIstart_positions end_positionsrurrrvc  Cs | dur | n|jj} |j|||||||| | dd } | d}||}|jddd\}}|d}|d}d}|dur | dur t|dkr|d}t| dkr| d} |d}| d|}| d|} t |d}|||}||| }||d }t |||| j | j d S) a4 bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. Example: In the example below, we prepare a question + context pair for the LayoutLM model. It will give us a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image). ```python >>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering >>> from datasets import load_dataset >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa", add_prefix_space=True) >>> model = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa", revision="1e3ebac") >>> dataset = load_dataset("nielsr/funsd", split="train") >>> example = dataset[0] >>> question = "what's his name?" >>> words = example["words"] >>> boxes = example["bboxes"] >>> encoding = tokenizer( ... question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt" ... ) >>> bbox = [] >>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)): ... if s == 1: ... bbox.append(boxes[w]) ... elif i == tokenizer.sep_token_id: ... bbox.append([1000] * 4) ... else: ... bbox.append([0] * 4) >>> encoding["bbox"] = torch.tensor([bbox]) >>> word_ids = encoding.word_ids(0) >>> outputs = model(**encoding) >>> loss = outputs.loss >>> start_scores = outputs.start_logits >>> end_scores = outputs.end_logits >>> start, end = word_ids[start_scores.argmax(-1)], word_ids[end_scores.argmax(-1)] >>> print(" ".join(words[start : end + 1])) M. Hamann P. Harper, P. Martinez ```NTrrrr!r)Z ignore_indexrA)r start_logits end_logitsrtr)r9rrrsplitrrdrrBclamprrrtr)r8rFrGrXrHr rZrIrrrurrrxrrrrZ total_lossZ ignored_indexrZ start_lossZend_lossr<r<r=rMsLB           z$LayoutLMForQuestionAnswering.forward)T) NNNNNNNNNNNN)rNrOrPr$rrrrr5rrzr{rr|rrMrRr<r<r:r=rs> r)rrrrrr)rSN)@rQtypingrrrr5Ztorch.utils.checkpointrZtorch.nnrrrZ activationsr Zmodeling_layersr Zmodeling_outputsr r rrrrZmodeling_utilsrrZ pytorch_utilsrrrutilsrrrZconfiguration_layoutlmrZ get_loggerrNloggerr1r/Modulerryfloatrhrir}rrrrrrrrrrrrrrr__all__r<r<r<r=st      T ;.)2 xm