Python Transformers library (NLP processing library) case code explanation

pip install transformers torch datasets

from transformers import pipeline
# Use sentiment analysis assembly lineclassifier = pipeline("sentiment-analysis")
result = classifier("I love programming with Transformers!")
print(result)  # [{'label': 'POSITIVE', 'score': 0.9998}]

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
text = "Hello, world!"
encoded = tokenizer(text, 
                    padding=True, 
                    truncation=True, 
                    return_tensors="pt")  # Return PyTorch tensorprint(encoded)
# {'input_ids': tensor([[101, 7592, 1010, 2088, 999, 102]]), 
#  'attention_mask': tensor([[1, 1, 1, 1, 1, 1]])}

from transformers import AutoModel
model = AutoModel.from_pretrained("bert-base-uncased")
outputs = model(**encoded)  # Forward communicationlast_hidden_states = outputs.last_hidden_state

from transformers import BertForSequenceClassification, Trainer, TrainingArguments
from datasets import load_dataset
# Load the datasetdataset = load_dataset("imdb")
tokenized_datasets = (
    lambda x: tokenizer(x["text"], padding=True, truncation=True),
    batched=True
)
# Define the modelmodel = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=2)
# Training parameter configurationtraining_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=8,
    evaluation_strategy="epoch"
)
# Trainer configurationtrainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["test"]
)
# Start training()

model.save_pretrained("./my_model")
tokenizer.save_pretrained("./my_model")
# Load custom modelnew_model = AutoModel.from_pretrained("./my_model")

from transformers import BertModel, BertTokenizer
import torch
model = BertModel.from_pretrained("bert-base-uncased", output_attentions=True)
inputs = tokenizer("The cat sat on the mat", return_tensors="pt")
outputs = model(**inputs)
# Extract attention weights from layer 0attention = [0][0]
print()  # [num_heads, seq_len, seq_len]

from transformers import TrainingArguments
training_args = TrainingArguments(
    fp16=True,  # Enable mixing accuracy    ...
)

from transformers import pipeline
# Load NER pipelinener_pipeline = pipeline("ner", model="dslim/bert-base-NER")
text = "Apple was founded by Steve Jobs in Cupertino."
results = ner_pipeline(text)
# Results visualizationfor entity in results:
    print(f"{entity['word']} -> {entity['entity']} (confidence: {entity['score']:.2f})")

from transformers import TrainingArguments
training_args = TrainingArguments(
    output_dir="./results",
    learning_rate=2e-5,
    weight_decay=0.01,
    warmup_steps=500,          # Number of learning rate warm-up steps    gradient_accumulation_steps=2,  # Gradient accumulation (save video memory)    gradient_clipping=1.0,     # Gradient crop threshold    ...
)

import torch
from transformers import BertForSequenceClassification
class CustomModel(BertForSequenceClassification):
    def __init__(self, config):
        super().__init__(config)
    def forward(self, input_ids, attention_mask, labels=None):
        outputs = super().forward(input_ids, attention_mask)
        logits = 
        if labels is not None:
            loss_fct = (weight=([1.0, 2.0]))  # Category weight            loss = loss_fct((-1, 2), (-1))
            return {"loss": loss, "logits": logits}
        return outputs

from transformers import GPT2LMHeadModel, GPT2Tokenizer
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")
prompt = "In a world where AI dominates,"
input_ids = (prompt, return_tensors="pt")
# Generate text (configure generation parameters)output = (
    input_ids, 
    max_length=100, 
    temperature=0.7,        # Control randomness (low value is more certain)    top_k=50,               # Limit the number of candidate words    num_return_sequences=3  # Generate 3 different results)
for seq in output:
    print((seq, skip_special_tokens=True))

from transformers import pipeline
qa_pipeline = pipeline("question-answering", model="deepset/roberta-base-squad2")
context = """
Hugging Face is a company based in New York City. 
Its Transformers library is widely used in NLP.
"""
question = "Where is Hugging Face located?"
result = qa_pipeline(question=question, context=context)
print(f"Answer: {result['answer']} (score: {result['score']:.2f})")
# Answer: New York City (score: 0.92)

from transformers import BertModel, AutoTokenizer
import torch
model = BertModel.from_pretrained("bert-base-uncased")
quantized_model = .quantize_dynamic(
    model, 
    {},   # Quantify all linear layers    dtype=torch.qint8
)
# Increased inference speed after quantization2-4Double，Model volume reduction is about75%

from transformers import BertTokenizer, BertForSequenceClassification
from  import export
model = BertForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
# Example inputdummy_input = tokenizer("This is a test", return_tensors="pt")
# Export as ONNXexport(
    model,
    (dummy_input["input_ids"], dummy_input["attention_mask"]),
    "",
    opset_version=13,
    input_names=["input_ids", "attention_mask"],
    output_names=["logits"],
    dynamic_axes={"input_ids": {0: "batch"}, "attention_mask": {0: "batch"}}
)

import torch
# Insert video memory monitoring in the training loopprint(f"Allocated: {.memory_allocated() / 1e9:.2f} GB")
print(f"Cached: {.memory_reserved() / 1e9:.2f} GB")

from  import profile, record_function, ProfilerActivity
with profile(activities=[], record_shapes=True) as prof:
    outputs = model(**inputs)
print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=10))

from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
tokenizer = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
# Chinese to Englishtokenizer.src_lang = "zh_CN"
text = "Welcome to Transformers library"
encoded = tokenizer(text, return_tensors="pt")
generated_tokens = (**encoded, forced_bos_token_id=tokenizer.lang_code_to_id["en_XX"])
print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True))
# ['Welcome to the Transformers library']

from PIL import Image
from transformers import CLIPProcessor, CLIPModel
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
image = ("")
text = ["a photo of a cat", "a photo of a dog"]
inputs = processor(text=text, images=image, return_tensors="pt", padding=True)
outputs = model(**inputs)
# Calculate the similarity of the graphiclogits_per_image = outputs.logits_per_image
probs = logits_per_image.softmax(dim=1)  # Probability distribution

import  as nn
class TransformerBlock():
    def __init__(self, d_model=512, nhead=8):
        super().__init__()
         = (d_model, nhead)
         = (d_model, d_model)
         = (d_model)
    def forward(self, x):
        attn_output, _ = (x, x, x)
        x = x + attn_output
        x = (x)
        x = x + (x)
        return x

from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
# Add special vocabulary manuallytokenizer.add_tokens(["【Special Word】"])
# Adjust the model embedding layermodel.resize_token_embeddings(len(tokenizer)) 

from transformers import LlamaForCausalLM, LlamaTokenizer, TrainingArguments
# Loading the model and word participle (requiring permission)model = LlamaForCausalLM.from_pretrained("decapoda-research/llama-7b-hf")
tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
# Low-rank adaptation (LoRA) fine adjustmentfrom peft import get_peft_model, LoraConfig
lora_config = LoraConfig(
    r=8,  # Low rank dimension    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],  # Only fine-tune some modules    lora_dropout=0.05,
    bias="none"
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # Show the proportion of trainable parameters (usually <1%)# Continue to configure training parameters...

# Use the TRL library for RLHF trainingfrom trl import PPOTrainer, AutoModelForCausalLMWithValueHead
model = AutoModelForCausalLMWithValueHead.from_pretrained("gpt2")
ppo_trainer = PPOTrainer(
    model=model,
    config=training_args,
    dataset=dataset,
    tokenizer=tokenizer
)
# Define the reward modelfor epoch in range(3):
    for batch in ppo_trainer.dataloader:
        # Generate a response        response_tensors = (batch["input_ids"])
        # Calculate rewards (need to custom reward functions)        rewards = calculate_rewards(response_tensors, batch)
        # PPO optimization steps        ppo_trainer.step(
            response_tensors,
            rewards,
            batch["attention_mask"]
        )

from transformers import TrainingArguments
# Configure distributed trainingtraining_args = TrainingArguments(
    per_device_train_batch_size=4,
    gradient_accumulation_steps=8,
    fp16=True,
    tpu_num_cores=8,  # Specify the number of cores when using TPU    dataloader_num_workers=4,
    deepspeed="./configs/deepspeed_config.json"  # Optimize with DeepSpeed)
# DeepSpeed ​​configuration file example (ds_config.json):{
  "fp16": {
    "enabled": true
  },
  "optimizer": {
    "type": "AdamW",
    "params": {
      "lr": 3e-5
    }
  },
  "zero_optimization": {
    "stage": 3  # Enable ZeRO-3 optimization  }
}

from fastapi import FastAPI
from pydantic import BaseModel
from transformers import pipeline
app = FastAPI()
generator = pipeline("text-generation", model="gpt2")
class Request(BaseModel):
    text: str
    max_length: int = 100
@("/generate")
async def generate_text(request: Request):
    result = generator(, max_length=request.max_length)
    return {"generated_text": result[0]["generated_text"]}
# Start the service：uvicorn main:app --port 8000

from transformers import AutoTokenizer, AutoModelForQuestionAnswering
tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
def process_long_text(context, question, max_length=384, stride=128):
    # Process long text in chunks    inputs = tokenizer(
        question,
        context,
        max_length=max_length,
        truncation="only_second",
        stride=stride,
        return_overflowing_tokens=True,
        return_offsets_mapping=True
    )
    # Reason and merge results for each block    best_score = 0
    best_answer = ""
    for i in range(len(inputs["input_ids"])):
        outputs = model(**{k: ([v[i]]) for k, v in ()})
        answer_start = (outputs.start_logits)
        answer_end = (outputs.end_logits) + 1
        score = (outputs.start_logits[answer_start] + outputs.end_logits[answer_end-1]).item()
        if score > best_score:
            best_score = score
            best_answer = (inputs["input_ids"][i][answer_start:answer_end])
    return best_answer

# Cross-language migration using XLM-RoBERTafrom transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification
tokenizer = XLMRobertaTokenizer.from_pretrained("xlm-roberta-base")
model = XLMRobertaForSequenceClassification.from_pretrained("xlm-roberta-base")
# Fine-tuning with a small number of samples（Code andBERTTraining similar）

from  import LayerIntegratedGradients
from transformers import BertForSequenceClassification
model = BertForSequenceClassification.from_pretrained("bert-base-uncased")
def forward_func(input_ids, attention_mask):
    return model(input_ids, attention_mask).logits
lig = LayerIntegratedGradients(forward_func, )
# Calculate the importance of input wordsattributions, delta = (
    inputs=input_ids,
    baselines=tokenizer.pad_token_id * torch.ones_like(input_ids),
    additional_forward_args=attention_mask,
    return_convergence_delta=True
)
# Visualize the resultsimport  as plt
(range(len(attributions[0])), attributions[0].detach().numpy())
(ticks=range(len(tokens)), labels=tokens, rotation=90)
()

import spacy
from spacy_transformers import TransformersLanguage, TransformersWordPiecer
# Create a spacy pipelinenlp = TransformersLanguage(trf_name="bert-base-uncased")
# Custom components@("CustomClassifier.v1")
def create_classifier(transformer, tok2vec, n_classes):
    return TransformersTextCategorizer(transformer, tok2vec, n_classes)
# Use Transformer model directly in spacydoc = nlp("This is a text to analyze.")
print(doc._.trf_last_hidden_state.shape)  # [seq_len, hidden_dim]

import gradio as gr
from transformers import pipeline
ner_pipeline = pipeline("ner")
def extract_entities(text):
    results = ner_pipeline(text)
    return {"text": text, "entities": [
        {"entity": res["entity"], "start": res["start"], "end": res["end"]}
        for res in results
    ]}
(
    fn=extract_entities,
    inputs=(lines=5),
    outputs=()
).launch()

# Use nn_pruning for structured pruningfrom transformers import BertForSequenceClassification
from nn_pruning import ModelPruning
model = BertForSequenceClassification.from_pretrained("bert-base-uncased")
pruner = ModelPruning(
    model,
    target_sparsity=0.5,  # Prune 50% attention head    pattern="block_sparse"  # Structured pruning pattern)
# Perform pruning and fine-tuningpruned_model = ()
pruned_model.save_pretrained("./pruned_bert")
# Knowledge distillation (Teacher → Student Model)from transformers import DistilBertForSequenceClassification, DistilBertTokenizer
teacher = BertForSequenceClassification.from_pretrained("bert-base-uncased")
student = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased")
# Use the distillation trainerfrom transformers import DistillationTrainingArguments, DistillationTrainer
training_args = DistillationTrainingArguments(
    output_dir="./distilled",
    temperature=2.0,  # Softening probability distribution    alpha_ce=0.5,     # Cross entropy loss weight    alpha_mse=0.5     # Hide layer MSE loss weight)
trainer = DistillationTrainer(
    teacher=teacher,
    student=student,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    tokenizer=tokenizer
)
()

# Convert the model to TensorRT enginetrtexec --onnx= --saveEngine= --fp16

# Python calls TensorRT engineimport tensorrt as trt
import  as cuda
runtime = (())
with open("", "rb") as f:
    engine = runtime.deserialize_cuda_engine(())
context = engine.create_execution_context()
# Bind the input and output buffer for inference

from transformers import AutoTokenizer, AutoModelForTokenClassification
tokenizer = AutoTokenizer.from_pretrained("dmis-lab/biobert-v1.1")
model = AutoModelForTokenClassification.from_pretrained("dmis-lab/biobert-v1.1")
text = "The patient exhibited EGFR mutations and responded to osimertinib."
inputs = tokenizer(text, return_tensors="pt")
outputs = model(**inputs).logits
# Extract gene entitiespredictions = (outputs, dim=2)
print([([token]) for token in inputs.input_ids[0]])
print(())  # BIOMark the results

# Contract Terms Classificationfrom transformers import BertTokenizer, BertForSequenceClassification
tokenizer = BertTokenizer.from_pretrained("nlpaueb/legal-bert-base-uncased")
model = BertForSequenceClassification.from_pretrained("nlpaueb/legal-bert-base-uncased")
clause = "The Parties hereby agree to arbitrate all disputes in accordance with ICC rules."
inputs = tokenizer(clause, return_tensors="pt", truncation=True, padding=True)
outputs = model(**inputs)
predicted_class = ().item()  # 0: Arbitration clause, 1: Confidentiality clauses, etc.

from transformers import BertForSequenceClassification
import coremltools as ct
model = BertForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
# Transformation Modeltraced_model = (model, (input_ids, attention_mask))
mlmodel = (
    traced_model,
    inputs=[
        (name="input_ids", shape=input_ids.shape),
        (name="attention_mask", shape=attention_mask.shape)
    ]
)
("")

from transformers import TFBertForSequenceClassification
import tensorflow as tf
model = TFBertForSequenceClassification.from_pretrained("bert-base-uncased")
# Convert to TFLiteconverter = .from_keras_model(model)
 = []  # Dynamic range quantizationtflite_model = ()
with open("model_quant.tflite", "wb") as f:
    (tflite_model)

from transformers import pipeline
from fairness_metrics import demographic_parity
# Detect model biasclassifier = pipeline("text-classification", model="bert-base-uncased")
protected_groups = {
    "gender": ["she", "he"],
    "race": ["African", "European"]
}
bias_scores = {}
for category, terms in protected_groups.items():
    texts = [f"{term} is qualified for this position" for term in terms]
    results = classifier(texts)
    bias_scores[category] = demographic_parity(results)

from textattack import AttackRecipe
from  import HuggingFaceModelWrapper
model_wrapper = HuggingFaceModelWrapper(model, tokenizer)
attack = ("bae")  # BAE attack method# Generate adversarial samplesattack_args = (num_examples=5)
attacker = (attack, model_wrapper, attack_args)
attack_results = attacker.attack_dataset(dataset)

from transformers import LongformerModel
model = LongformerModel.from_pretrained("allenai/longformer-base-4096")
inputs = tokenizer("This is a very long document..."*1000, return_tensors="pt")
outputs = model(**inputs)  # Longest support4096 tokens

# Use Switch Transformersfrom transformers import SwitchTransformersForConditionalGeneration
model = SwitchTransformersForConditionalGeneration.from_pretrained("google/switch-base-8")
outputs = (
    input_ids,
    expert_choice_mask=True,  # Track expert routing)
print(outputs.expert_choices)  # Show eachtokenExperts using it

"""
 End-to-end text classification system architecture:
 1. Data acquisition → 2. Cleaning → 3. Annotation → 4. Model training → 5. Evaluation → 6. Deployment → 7. Monitoring
 """
# Enhanced training process in step 4from transformers import TrainerCallback
class CustomCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        # Record metrics in real time to Prometheus        prometheus_logger.log_metrics(logs)
#Drift detection in step 7from alibi_detect.cd import MMDDrift
detector = MMDDrift(
    X_train, 
    backend="tensorflow", 
    p_val=0.05
)
drift_preds = (X_prod)