baseline与DataWhale的步骤一起运行。

1. 环境安装。

   !pip install simple-aesthetics-predictor!pip install -v -e data-juicer!pip uninstall pytorch-lightning -y!pip install peft lightning pandas torchvision!pip install -e DiffSynth-Studio。

2. 下载数据集。

   #下载from的数据集 modelscope.msdatasets import MsDatasetds = MsDataset.load(    'AI-ModelScope/lowres_anime',    subset_name='default',    split='train',    cache_dir="/mnt/workspace/kolors/data")import json, osfrom data_juicer.utils.mm_utils import SpecialTokensfrom tqdm import tqdmos.makedirs("./data/lora_dataset/train", exist_ok=True)os.makedirs("./data/data-juicer/input", exist_ok=True)with open("./data/data-juicer/input/metadata.jsonl", "w") as f:    for data_id, data in enumerate(tqdm(ds)):        image = data["image"].convert("RGB")        image.save(f"/mnt/workspace/kolors/data/lora_dataset/train/{ data_id}.jpg")        metadata = { "text": "二次元", "image": [f"/mnt/workspace/kolors/data/lora_dataset/train/{ data_id}.jpg"]}        f.write(json.dumps(metadata))        f.write("\n")。

3. #xfff0处理数据集c;保存数据处理结果。

   data_juicer_config = """# global parametersproject_name: 'data-process'dataset_path: './data/data-juicer/input/metadata.jsonl'  # path to your dataset directory or filenp: 4  # number of subprocess to process your datasettext_keys: 'text'image_key: 'image'image_special_token: '<__dj__image>'export_path: './data/data-juicer/output/result.jsonl'# process schedule# a list of several process operators with their argumentsprocess:    - image_shape_filter:        min_width: 1024        min_height: 1024        any_or_all: any    - image_aspect_ratio_filter:        min_ratio: 0.5        max_ratio: 2.0        any_or_all: any"""with open("data/data-juicer/data_juicer_config.yaml", "w") as file:    file.write(data_juicer_config.strip())!dj-process --config data/data-juicer/data_juicer_config.yamlimport pandas as pdimport os, jsonfrom PIL import Imagefrom tqdm import tqdmtexts, file_names = [], []os.makedirs("./data/lora_dataset_processed/train", exist_ok=True)with open("./data/data-juicer/output/result.jsonl", "r") as file:    for data_id, data in enumerate(tqdm(file.readlines())):        data = json.loads(data)        text = data["text"]        texts.append(text)        image = Image.open(data["image"][0])        image_path = f"./data/lora_dataset_processed/train/{ data_id}.jpg"        image.save(image_path)        file_names.append(f"{ data_id}.jpg")data_frame = pd.DataFrame()data_frame["file_name"] = file_namesdata_frame["text"] = textsdata_frame.to_csv("./data/lora_dataset_processed/train/metadata.csv", index=False, encoding="utf-8-sig")data_frame。

4. lora微调。

   # 下载模型from diffsynth import download_modelsdownload_models(["Kolors", "SDXL-vae-fp16-fix"])#import模型训练 oscmd = """python DiffSynth-Studio/examples/train/kolors/train_kolors_lora.py \  --pretrained_unet_path models/kolors/Kolors/unet/diffusion_pytorch_model.safetensors \  --pretrained_text_encoder_path models/kolors/Kolors/text_encoder \  --pretrained_fp16_vae_path models/sdxl-vae-fp16-fix/diffusion_pytorch_model.safetensors \  --lora_rank 16 \  --lora_alpha 4.0 \  --dataset_path data/lora_dataset_processed \  --output_path ./models \  --max_epochs 1 \  --center_crop \  --use_gradient_checkpointing \  --precision "16-mixed"""".strip()os.system(cmd)。

5. 加载微调模型。

   from diffsynth import ModelManager, SDXLImagePipelinefrom peft import LoraConfig, inject_adapter_in_modelimport torchdef load_lora(model, lora_rank, lora_alpha, lora_path):    lora_config = LoraConfig(        r=lora_rank,        lora_alpha=lora_alpha,        init_lora_weights="gaussian",        target_modules=["to_q", "to_k", "to_v", "to_out"],    )    model = inject_adapter_in_model(lora_config, model)    state_dict = torch.load(lora_path, map_location="cpu")    model.load_state_dict(state_dict, strict=False)    return model# Load modelsmodel_manager = ModelManager(torch_dtype=torch.float16， device="cuda",                             file_path_list=[                                 "models/kolors/Kolors/text_encoder",                                 "models/kolors/Kolors/unet/diffusion_pytorch_model.safetensors",                                 "models/kolors/Kolors/vae/diffusion_pytorch_model.safetensors"                             ])pipe = SDXLImagePipeline.from_model_manager(model_manager)# Load LoRApipe.unet = load_lora(    pipe.unet,    lora_rank=16, # This parameter should be consistent with that in your training script.    lora_alpha=2.0, # lora_alpha can control the weight of LoRA.    lora_path="models/lightning_logs/version_0/checkpoints/epoch=0-step=500.ckpt")。

6. 图片生成（修改提示词）

   torch.manual_seed(0)image = pipe(    prompt="一只可爱的小象穿着探险帽，穿狩猎服，站在沙漠地面上󿀌鼻子高举󿀌Pixar的风格 animation。",    negative_prompt="丑陋、变形、嘈杂、模糊、低对比度",    cfg_scale=4,    num_inference_steps=50, height=1024, width=1024,)image.save("1.jpg")。

   感觉：感受到提示词的使用，并体验了生成图片的过程，作为一名大一新生，我觉得很新奇。