فراتر از تصور: پتانسیل هوش مصنوعی در هنر دیجیتال

import torch
import os
from glob import glob
from torch import nn, optim
from torch.utils.data import DataLoader, Dataset
from torchvision import models, transforms
from transformers import VisionEncoderDecoderModel, ViTImageProcessor, AutoModel, AutoTokenizer
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from rich import print as rp
import wandb

wandb.init(project="spritemaker", entity="goldenkooy")

class TextEncoder:
    def __init__(self, model_name="bert-base-uncased"):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir="./models")
        self.model = AutoModel.from_pretrained(model_name, cache_dir="./models")
        self.model.eval()

    def encode_text(self, text):
        with torch.no_grad():
            inputs = self.tokenizer(text, return_tensors="pt", padding=True, truncation=True)
            outputs = self.model(**inputs)
        return outputs.last_hidden_state[:, 0, :]

class ImageEncoder:
    def __init__(self):
        self.model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)
        self.model.eval()
        self.transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        ])

    def encode_image(self, image_path):
        image = Image.open(image_path).convert('RGB')
        image = self.transform(image)
        image = image.unsqueeze(0)  # Add batch dimension
        with torch.no_grad():
            features = self.model(image)
        return features

class SpriteTextDataset(Dataset):
    def __init__(self, image_dir, text_dir, image_encoder, text_encoder):
        self.image_encoder = image_encoder
        self.text_encoder = text_encoder
        self.data = []

        # Load all image paths
        image_paths = glob(os.path.join(image_dir, '*.png'))

        # Debug: print the found image paths
        rp(f"Found image paths: {image_paths}")

        # Load descriptions and pair them with images
        for image_path in image_paths:
            base_filename = os.path.splitext(os.path.basename(image_path))[0]
            text_path = os.path.join(text_dir, f"{base_filename}.txt")
            if os.path.exists(text_path):
                with open(text_path, 'r', encoding='utf-8') as file:
                    description = file.read().strip()
                    self.data.append((image_path, description))
            else:
                rp(f"Warning: No description file found for {image_path}")

        # Debug: print the dataset size
        rp(f"Dataset size: {len(self.data)}")

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        image_path, description = self.data[idx]
        image_features = self.image_encoder.encode_image(image_path)
        text_features = self.text_encoder.encode_text(description)
        return image_features, text_features

class SpriteTextDataLoader:
    def __init__(self, dataset, batch_size, num_workers):
        self.dataset = dataset
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers)

    def __iter__(self):
        return iter(self.data_loader)

    def __len__(self):
        return len(self.data_loader)

data_loader = SpriteTextDataLoader(sprite_text_dataset, batch_size=32, num_workers=4)
for batch in data_loader:
    images, texts = batch
    # Train our model on the batch

class SpriteMaker(nn.Module):
    def __init__(self, text_encoder, image_encoder, sprite_generator):
        super(SpriteMaker, self).__init__()
        self.text_encoder = text_encoder
        self.image_encoder = image_encoder
        self.sprite_generator = sprite_generator

    def forward(self, text, image):
        text_features = self.text_encoder(text)
        image_features = self.image_encoder(image)
        sprite_features = torch.cat((text_features, image_features), dim=1)
        sprite = self.sprite_generator(sprite_features)
        return sprite

def train_model(model, data_loader, optimizer, loss_fn):
    model.train()
    total_loss = 0
    for batch in data_loader:
        images, texts = batch
        images = images.to(device)
        texts = texts.to(device)
        optimizer.zero_grad()
        outputs = model(texts, images)
        loss = loss_fn(outputs, targets)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
    print(f"Training loss: {total_loss / len(data_loader)}")

loss_fn = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
Training the Model

We'll train our model for 5 epochs using the train_model function.
We'll print the training loss at each epoch.



for epoch in range(5):
    train_model(model, data_loader, optimizer, loss_fn)
    print(f"Epoch {epoch+1}, Training loss: {total_loss / len(data_loader)}")

def evaluate_model(model, test_loader):
    model.eval()
    total_loss = 0
    correct = 0
    with torch.no_grad():
        for batch in test_loader:
            images, texts = batch
            images = images.to(device)
            texts = texts.to(device)
            outputs = model(texts, images)
            loss = loss_fn(outputs, targets)
            total_loss += loss.item()
            _, predicted = torch.max(outputs, 1)
            correct += (predicted == targets).sum().item()
    accuracy = correct / len(test_loader.dataset)
    print(f"Test Loss: {total_loss / len(test_loader)}")
    print(f"Test Accuracy: {accuracy:.2f}")

test_loss, test_accuracy = evaluate_model(model, test_loader)
print(f"Test Loss: {test_loss:.2f}")
print(f"Test Accuracy: {test_accuracy:.2f}")

def generate_sprite(model, text, image):
    model.eval()
    text_features = text_encoder(text)
    image_features = image_encoder(image)
    sprite_features = torch.cat((text_features, image_features), dim=1)
    sprite = model.sprite_generator(sprite_features)
    sprite = sprite.cpu().numpy()
    sprite = Image.fromarray(sprite)
    sprite.save("generated_sprite.png")

text = "Hello, world!"
image = Image.open("image.png")
generate_sprite(model, text, image)

import torch
import os
from glob import glob
from torch import nn, optim
from torch.utils.data import DataLoader, Dataset
from torchvision import models, transforms
from transformers import VisionEncoderDecoderModel, ViTImageProcessor, AutoModel, AutoTokenizer
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from rich import print as rp
import wandb

# Initialize WandB
wandb.init(project="spritemaker", entity="goldenkooy")

# Text encoder class
class TextEncoder:
    def __init__(self, model_name="bert-base-uncased"):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir="./models")
        self.model = AutoModel.from_pretrained(model_name, cache_dir="./models")
        self.model.eval()

    def encode_text(self, text):
        with torch.no_grad():
            inputs = self.tokenizer(text, return_tensors="pt", padding=True, truncation=True)
            outputs = self.model(**inputs)
        return outputs.last_hidden_state[:, 0, :]

# Image encoder class
class ImageEncoder:
    def __init__(self):
        self.model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)
        self.model.eval()
        self.transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        ])

    def encode_image(self, image_path):
        image = Image.open(image_path).convert('RGB')
        image = self.transform(image)
        image = image.unsqueeze(0)  # Add batch dimension
        with torch.no_grad():
            features = self.model(image)
        return features

# Sprite and text dataset class
class SpriteTextDataset(Dataset):
    def __init__(self, image_dir, text_dir, image_encoder, text_encoder):
        self.image_encoder = image_encoder
        self.text_encoder = text_encoder
        self.data = []

        # Load all image paths
        image_paths = glob(os.path.join(image_dir, '*.png'))

        # Debug: rp the found image paths
        rp(f"Found image paths: {image_paths}")

        # Load descriptions and pair them with images
        for image_path in image_paths:
            base_filename = os.path.splitext(os.path.basename(image_path))[0]
            text_path = os.path.join(text_dir, f"{base_filename}.txt")
            if os.path.exists(text_path):
                with open(text_path, 'r', encoding='utf-8') as file:
                    description = file.read().strip()
                    self.data.append((image_path, description))
            else:
                rp(f"Warning: No description file found for {image_path}")

        # Debug: rp the dataset size
        rp(f"Dataset size: {len(self.data)}")

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        image_path, text = self.data[idx]
        image_features = self.image_encoder.encode_image(image_path)
        text_features = self.text_encoder.encode_text(text)

        combined_features = torch.cat((image_features, text_features), dim=1)
        return combined_features

# Descriptor class for generating descriptions
class Descriptor:
    def __init__(self, cache_dir="./models"):
        self.model = VisionEncoderDecoderModel.from_pretrained(
            "nlpconnect/vit-gpt2-image-captioning", cache_dir=cache_dir
        )
        self.feature_extractor = ViTImageProcessor.from_pretrained(
            "nlpconnect/vit-gpt2-image-captioning", cache_dir=cache_dir
        )
        self.tokenizer = AutoTokenizer.from_pretrained(
            "nlpconnect/vit-gpt2-image-captioning", cache_dir=cache_dir
        )

        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.model.to(self.device)

        self.max_length = 16 
        self.num_beams = 4
        self.gen_kwargs = {"max_length": self.max_length, "num_beams": self.num_beams}

    def describe_image(self, image_path):
        image = Image.open(image_path)
        if image.mode != "RGB":
            image = image.convert(mode="RGB")

        pixel_values = self.feature_extractor(images=image, return_tensors="pt").pixel_values
        pixel_values = pixel_values.to(self.device)

        output_ids = self.model.generate(pixel_values, **self.gen_kwargs)
        description = self.tokenizer.decode(output_ids[0], skip_special_tokens=True).strip()
        return description

# VAE model class
class VAE(nn.Module):
    def __init__(self, input_dim, latent_dim):
        super(VAE, self).__init__()
        self.fc1 = nn.Linear(input_dim, 512)
        self.fc21 = nn.Linear(512, latent_dim)
        self.fc22 = nn.Linear(512, latent_dim)
        self.fc3 = nn.Linear(latent_dim, 512)
        self.fc4 = nn.Linear(512, input_dim)

    def encode(self, x):
        h1 = torch.relu(self.fc1(x))
        return self.fc21(h1), self.fc22(h1)

    def reparameterize(self, mu, logvar):
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        return mu + eps * std

    def decode(self, z):
        h3 = torch.relu(self.fc3(z))
        return torch.sigmoid(self.fc4(h3))

    def forward(self, x):
        mu, logvar = self.encode(x)
        z = self.reparameterize(mu, logvar)
        return self.decode(z), mu, logvar

# Utility functions for training and visualization
class Utils:
    def __init__(self, dataset, model, optimizer, text_encoder, checkpoint_dir="checkpoints"):
        self.train_data = dataset
        self.model = model
        self.optimizer = optimizer
        self.checkpoint_dir = checkpoint_dir
        self.text_encoder = text_encoder
        os.makedirs(checkpoint_dir, exist_ok=True)

    def save_checkpoint(self, epoch, loss):
        checkpoint_path = os.path.join(self.checkpoint_dir, 'latest_checkpoint.pth')
        checkpoint = {
            'epoch': epoch,
            'model_state_dict': self.model.state_dict(),
            'optimizer_state_dict': self.optimizer.state_dict(),
            'loss': loss
        }
        torch.save(checkpoint, checkpoint_path)
        rp(f'Checkpoint saved at {checkpoint_path}')

    def load_checkpoint(self):
        checkpoint_path = os.path.join(self.checkpoint_dir, 'latest_checkpoint.pth')
        if os.path.exists(checkpoint_path):
            checkpoint = torch.load(checkpoint_path)
            self.model.load_state_dict(checkpoint['model_state_dict'])
            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
            epoch = checkpoint['epoch']
            loss = checkpoint['loss']
            rp(f'Checkpoint loaded from {checkpoint_path}, epoch {epoch}, loss {loss}')
            return epoch, loss
        else:
            rp(f'No checkpoint found at {checkpoint_path}')
            return None, None

    def visualize_reconstructions(self, device="cpu"):
        self.model.eval()
        with torch.no_grad():
            for i, data in enumerate(self.train_data):
                data = data.to(device)
                reconstructed, _, _ = self.model(data)
                original = data.detach().cpu().numpy()
                reconstructed = reconstructed.detach().cpu().numpy()

                # Separate the image and text features
                original_image_features = original[:, :1000]
                reconstructed_image_features = reconstructed[:, :1000]

                # For a single sample, visualize the original and reconstructed images
                plt.figure(figsize=(12, 6))
                plt.subplot(1, 2, 1)
                plt.title('Original Image')
                self.visualize_image(original_image_features[0])  # Visualize original image

                plt.subplot(1, 2, 2)
                plt.title('Reconstructed Image')
                self.visualize_image(reconstructed_image_features[0])  # Visualize reconstructed image

                plt.show()

                if i >= 10:
                    break

    def visualize_image(self, text_prompt, num_samples=1):
        # Encode the text prompt
        text_features = self.text_encoder.encode_text(text_prompt)

        # Generate random latent variables
        latent_variables = torch.randn(num_samples, self.model.latent_dim)

        # Concatenate text features with latent variables
        combined_features = torch.cat((latent_variables, text_features.expand(num_samples, -1)), dim=1)

        # Decode the combined features
        with torch.no_grad():
            generated_images = self.model.decode(combined_features)

        # Visualize the generated images
        for i in range(num_samples):
            plt.figure(figsize=(4, 4))
            plt.imshow(generated_images[i].reshape(224, 224))  # Reshape as per your image size
            plt.axis('off')
            plt.title(f'Generated Image {i+1}')
            plt.show()


    def train_vae(self, epochs=10, batch_size=32, learning_rate=1e-3):
        dataloader = DataLoader(self.train_data, batch_size=batch_size, shuffle=True)
        for epoch in range(epochs):
            self.model.train()
            for batch in dataloader:
                batch = batch.to(next(self.model.parameters()).device)
                self.optimizer.zero_grad()
                recon_batch, mu, logvar = self.model(batch)
                loss = self.loss_function(recon_batch, batch, mu, logvar)
                loss.backward()
                self.optimizer.step()
            rp(f'Epoch {epoch + 1}, Loss: {loss.item()}')
            self.save_checkpoint(epoch + 1, loss.item())
            # Log the current learning rate to WandB
            wandb.log({"learning_rate": self.optimizer.param_groups[0]['lr']}, step=epoch)

            # Log the loss for the epoch
            wandb.log({"epoch_loss": loss.item()}, step=epoch)

            # print(f'Epoch {epoch + 1}, Loss: {loss.item()}')

    def loss_function(self, recon_x, x, mu, logvar):
        MSE = nn.functional.mse_loss(recon_x, x, reduction='sum')
        KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
        return MSE + KLD

# Assuming the rest of the script remains the same
# Instantiate encoders
text_encoder = TextEncoder()
image_encoder = ImageEncoder()

# Descriptor for generating descriptions
descriptor = Descriptor()

# Paths
image_dir="./trainings_data/spritesheets"
text_dir="./trainings_data/texts"

# Functions to fetch missing items and extract filenames
def fetch_missing_items(list1, list2):
    set2 = set(list2)
    return [item for item in list1 if item not in set2]

def extract_filenames(paths):
    return [os.path.splitext(os.path.basename(path))[0] for path in paths]

# Get lists of text and image files
text_files = glob(os.path.join(text_dir, '*.txt'))
image_files = glob(os.path.join(image_dir, '*.png'))

# Extract just the filenames without extensions for comparison
text_names = extract_filenames(text_files)
image_names = extract_filenames(image_files)

# Find descriptions missing for images
missing_descriptions = fetch_missing_items(image_names, text_names)

# Generate and write descriptions for missing files
for missing_name in missing_descriptions:
    image_path = os.path.join(image_dir, f"{missing_name}.png")
    text_path = os.path.join(text_dir, f"{missing_name}.txt")

    try:
        description = descriptor.describe_image(image_path)
        with open(text_path, "w") as f:
            f.write(description)
        rp(f"Generated description for: {missing_name}")
    except Exception as e:
        rp(f"Error generating description for {missing_name}: {e}")

# Create dataset
dataset = SpriteTextDataset(image_dir, text_dir, image_encoder, text_encoder)

# Verify dataset size
rp(f"Final dataset size: {len(dataset)}")

# 1000 features from resnet-50 + 768 features from BERT = 1768 input dimensions
vae = VAE(input_dim=1768, latent_dim=70)

# Create an optimizer
optimizer = optim.Adam(vae.parameters(), lr=1e-3)

# Instantiate utilities
utils = Utils(dataset, vae, optimizer, text_encoder)

# Optionally load checkpoint
start_epoch, start_loss = utils.load_checkpoint()

# Train VAE
if start_epoch is None:
    start_epoch = 0  # If no checkpoint is found, start from epoch 0
utils.train_vae(epochs=600 - start_epoch)

# Visualize progress
utils.visualize_reconstructions()

# Usage
text_prompt = "A pixel art character with a blue hat"
utils.visualize_image(text_prompt)