/** * @fileoverview Control real time music with text prompts * @license * SPDX-License-Identifier: Apache-2.0 */ import {css, CSSResultGroup, html, LitElement, svg} from 'lit'; import {customElement, property, query, state} from 'lit/decorators.js'; import {classMap} from 'lit/directives/class-map.js'; import {styleMap} from 'lit/directives/style-map.js'; import { GoogleGenAI, MusicGenerationMode, type LiveMusicGenerationConfig, type LiveMusicServerMessage, type LiveMusicSession, } from '@google/genai'; import {decode, decodeAudioData} from './utils'; const ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY, apiVersion: 'v1alpha', }); let model = 'lyria-realtime-exp'; interface Prompt { readonly promptId: string; readonly color: string; text: string; weight: number; } type PlaybackState = 'stopped' | 'playing' | 'loading' | 'paused'; /** Throttles a callback to be called at most once per `freq` milliseconds. */ function throttle(func: (...args: unknown[]) => void, delay: number) { let lastCall = 0; return (...args: unknown[]) => { const now = Date.now(); const timeSinceLastCall = now - lastCall; if (timeSinceLastCall >= delay) { func(...args); lastCall = now; } }; } const PROMPT_TEXT_PRESETS = [ 'Heavy Metal', 'Thrash Metal', 'Death Metal', 'Black Metal', 'Power Metal', 'Doom Metal', 'Progressive Metal', 'Symphonic Metal', 'Metalcore', 'Groove Metal', 'Industrial Metal', 'Sludge Metal', 'Djent', 'Viking Metal', 'Folk Metal', 'Gothic Metal', ]; const COLORS = [ '#9900ff', '#5200ff', '#ff25f6', '#2af6de', '#ffdd28', '#3dffab', '#d8ff3e', '#d9b2ff', ]; function getUnusedRandomColor(usedColors: string[]): string { const availableColors = COLORS.filter((c) => !usedColors.includes(c)); if (availableColors.length === 0) { // If no available colors, pick a random one from the original list. return COLORS[Math.floor(Math.random() * COLORS.length)]; } return availableColors[Math.floor(Math.random() * availableColors.length)]; } // WeightSlider component // ----------------------------------------------------------------------------- /** A slider for adjusting and visualizing prompt weight. */ @customElement('weight-slider') class WeightSlider extends LitElement { static override styles = css` :host { cursor: ns-resize; position: relative; height: 100%; display: flex; justify-content: center; flex-direction: column; align-items: center; padding: 5px; } .scroll-container { width: 100%; flex-grow: 1; display: flex; flex-direction: column; justify-content: center; align-items: center; } .value-display { font-size: 1.3vmin; color: #ccc; margin: 0.5vmin 0; user-select: none; text-align: center; } .slider-container { position: relative; width: 10px; height: 100%; background-color: #0009; border-radius: 4px; } #thumb { position: absolute; bottom: 0; left: 0; width: 100%; border-radius: 4px; box-shadow: 0 0 3px rgba(0, 0, 0, 0.7); } `; @property({type: Number}) value = 0; // Range 0-2 @property({type: String}) color = '#000'; @query('.scroll-container') private scrollContainer!: HTMLDivElement; private dragStartPos = 0; private dragStartValue = 0; private containerBounds: DOMRect | null = null; constructor() { super(); this.handlePointerDown = this.handlePointerDown.bind(this); this.handlePointerMove = this.handlePointerMove.bind(this); this.handleTouchMove = this.handleTouchMove.bind(this); this.handlePointerUp = this.handlePointerUp.bind(this); } private handlePointerDown(e: PointerEvent) { e.preventDefault(); this.containerBounds = this.scrollContainer.getBoundingClientRect(); this.dragStartPos = e.clientY; this.dragStartValue = this.value; document.body.classList.add('dragging'); window.addEventListener('pointermove', this.handlePointerMove); window.addEventListener('touchmove', this.handleTouchMove, { passive: false, }); window.addEventListener('pointerup', this.handlePointerUp, {once: true}); this.updateValueFromPosition(e.clientY); } private handlePointerMove(e: PointerEvent) { this.updateValueFromPosition(e.clientY); } private handleTouchMove(e: TouchEvent) { e.preventDefault(); this.updateValueFromPosition(e.touches[0].clientY); } private handlePointerUp(e: PointerEvent) { window.removeEventListener('pointermove', this.handlePointerMove); document.body.classList.remove('dragging'); this.containerBounds = null; } private handleWheel(e: WheelEvent) { e.preventDefault(); const delta = e.deltaY; this.value = this.value + delta * -0.005; this.value = Math.max(0, Math.min(2, this.value)); this.dispatchInputEvent(); } private updateValueFromPosition(clientY: number) { if (!this.containerBounds) return; const trackHeight = this.containerBounds.height; // Calculate position relative to the top of the track const relativeY = clientY - this.containerBounds.top; // Invert and normalize (0 at bottom, 1 at top) const normalizedValue = 1 - Math.max(0, Math.min(trackHeight, relativeY)) / trackHeight; // Scale to 0-2 range this.value = normalizedValue * 2; this.dispatchInputEvent(); } private dispatchInputEvent() { this.dispatchEvent(new CustomEvent('input', {detail: this.value})); } override render() { const thumbHeightPercent = (this.value / 2) * 100; const thumbStyle = styleMap({ height: `${thumbHeightPercent}%`, backgroundColor: this.color, // Hide thumb if value is 0 or very close to prevent visual glitch display: this.value > 0.01 ? 'block' : 'none', }); const displayValue = this.value.toFixed(2); return html`
${displayValue}
`; } } // Base class for icon buttons. class IconButton extends LitElement { static override styles = css` :host { position: relative; display: flex; align-items: center; justify-content: center; pointer-events: none; } :host(:hover) svg { transform: scale(1.2); } svg { width: 100%; height: 100%; transition: transform 0.5s cubic-bezier(0.25, 1.56, 0.32, 0.99); } .hitbox { pointer-events: all; position: absolute; width: 65%; aspect-ratio: 1; top: 9%; border-radius: 50%; cursor: pointer; } ` as CSSResultGroup; // Method to be implemented by subclasses to provide the specific icon SVG protected renderIcon() { return svg``; // Default empty icon } private renderSVG() { return html` ${this.renderIcon()} `; } override render() { return html`${this.renderSVG()}
`; } } // PlayPauseButton // ----------------------------------------------------------------------------- /** A button for toggling play/pause. */ @customElement('play-pause-button') export class PlayPauseButton extends IconButton { @property({type: String}) playbackState: PlaybackState = 'stopped'; static override styles = [ IconButton.styles, css` .loader { stroke: #ffffff; stroke-width: 3; stroke-linecap: round; animation: spin linear 1s infinite; transform-origin: center; transform-box: fill-box; } @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(359deg); } } `, ]; private renderPause() { return svg``; } private renderPlay() { return svg``; } private renderLoading() { return svg``; } override renderIcon() { if (this.playbackState === 'playing') { return this.renderPause(); } else if (this.playbackState === 'loading') { return this.renderLoading(); } else { return this.renderPlay(); } } } @customElement('reset-button') export class ResetButton extends IconButton { private renderResetIcon() { return svg``; } override renderIcon() { return this.renderResetIcon(); } } // AddPromptButton component // ----------------------------------------------------------------------------- /** A button for adding a new prompt. */ @customElement('add-prompt-button') export class AddPromptButton extends IconButton { private renderAddIcon() { return svg``; } override renderIcon() { return this.renderAddIcon(); } } // Toast Message component // ----------------------------------------------------------------------------- @customElement('toast-message') class ToastMessage extends LitElement { static override styles = css` .toast { line-height: 1.6; position: fixed; top: 20px; left: 50%; transform: translateX(-50%); background-color: #000; color: white; padding: 15px; border-radius: 5px; display: flex; align-items: center; justify-content: space-between; gap: 15px; min-width: 200px; max-width: 80vw; transition: transform 0.5s cubic-bezier(0.19, 1, 0.22, 1); z-index: 11; } button { border-radius: 100px; aspect-ratio: 1; border: none; color: #000; cursor: pointer; } .toast:not(.showing) { transition-duration: 1s; transform: translate(-50%, -200%); } `; @property({type: String}) message = ''; @property({type: Boolean}) showing = false; override render() { return html`
${this.message}
`; } show(message: string) { this.showing = true; this.message = message; } hide() { this.showing = false; } } /** A single prompt input */ @customElement('prompt-controller') class PromptController extends LitElement { static override styles = css` .prompt { position: relative; height: 100%; width: 100%; display: flex; flex-direction: column; align-items: center; box-sizing: border-box; overflow: hidden; background-color: #2a2a2a; border-radius: 5px; } .remove-button { position: absolute; top: 1.2vmin; left: 1.2vmin; background: #666; color: #fff; border: none; border-radius: 50%; width: 2.8vmin; height: 2.8vmin; font-size: 1.8vmin; display: flex; align-items: center; justify-content: center; line-height: 2.8vmin; cursor: pointer; opacity: 0.5; transition: opacity 0.2s; z-index: 10; } .remove-button:hover { opacity: 1; } weight-slider { /* Calculate height: 100% of parent minus controls height and margin */ max-height: calc(100% - 9vmin); flex: 1; min-height: 10vmin; width: 100%; box-sizing: border-box; overflow: hidden; margin: 2vmin 0 1vmin; } .controls { display: flex; flex-direction: column; flex-shrink: 0; align-items: center; gap: 0.2vmin; width: 100%; height: 8vmin; padding: 0 0.5vmin; box-sizing: border-box; margin-bottom: 1vmin; } #text { font-family: 'Google Sans', sans-serif; font-size: 1.8vmin; width: 100%; flex-grow: 1; max-height: 100%; padding: 0.4vmin; box-sizing: border-box; text-align: center; word-wrap: break-word; overflow-y: auto; border: none; outline: none; -webkit-font-smoothing: antialiased; color: #fff; scrollbar-width: thin; scrollbar-color: #666 #1a1a1a; } #text::-webkit-scrollbar { width: 6px; } #text::-webkit-scrollbar-track { background: #0009; border-radius: 3px; } #text::-webkit-scrollbar-thumb { background-color: #666; border-radius: 3px; } :host([filtered='true']) #text { background: #da2000; } `; @property({type: String, reflect: true}) promptId = ''; @property({type: String}) text = ''; @property({type: Number}) weight = 0; @property({type: String}) color = ''; @query('weight-slider') private weightInput!: WeightSlider; @query('#text') private textInput!: HTMLSpanElement; private handleTextKeyDown(e: KeyboardEvent) { if (e.key === 'Enter') { e.preventDefault(); this.updateText(); (e.target as HTMLElement).blur(); } } private dispatchPromptChange() { this.dispatchEvent( new CustomEvent('prompt-changed', { detail: { promptId: this.promptId, text: this.text, weight: this.weight, color: this.color, }, }), ); } private updateText() { console.log('updateText'); const newText = this.textInput.textContent?.trim(); if (newText === '') { this.textInput.textContent = this.text; return; } this.text = newText; this.dispatchPromptChange(); } private updateWeight() { this.weight = this.weightInput.value; this.dispatchPromptChange(); } private dispatchPromptRemoved() { this.dispatchEvent( new CustomEvent('prompt-removed', { detail: this.promptId, bubbles: true, composed: true, }), ); } override render() { const classes = classMap({ 'prompt': true, }); return html`
${this.text}
`; } } /** A panel for managing real-time music generation settings. */ @customElement('settings-controller') class SettingsController extends LitElement { static override styles = css` :host { display: block; padding: 2vmin; background-color: #2a2a2a; color: #eee; box-sizing: border-box; border-radius: 5px; font-family: 'Google Sans', sans-serif; font-size: 1.5vmin; overflow-y: auto; scrollbar-width: thin; scrollbar-color: #666 #1a1a1a; transition: width 0.3s ease-out max-height 0.3s ease-out; } :host([showadvanced]) { max-height: 40vmin; } :host::-webkit-scrollbar { width: 6px; } :host::-webkit-scrollbar-track { background: #1a1a1a; border-radius: 3px; } :host::-webkit-scrollbar-thumb { background-color: #666; border-radius: 3px; } .setting { margin-bottom: 0.5vmin; display: flex; flex-direction: column; gap: 0.5vmin; } label { font-weight: bold; display: flex; justify-content: space-between; align-items: center; white-space: nowrap; user-select: none; } label span:last-child { font-weight: normal; color: #ccc; min-width: 3em; text-align: right; } input[type='range'] { --track-height: 8px; --track-bg: #0009; --track-border-radius: 4px; --thumb-size: 16px; --thumb-bg: #5200ff; --thumb-border-radius: 50%; --thumb-box-shadow: 0 0 3px rgba(0, 0, 0, 0.7); --value-percent: 0%; -webkit-appearance: none; appearance: none; width: 100%; height: var(--track-height); background: transparent; cursor: pointer; margin: 0.5vmin 0; border: none; padding: 0; vertical-align: middle; } input[type='range']::-webkit-slider-runnable-track { width: 100%; height: var(--track-height); cursor: pointer; border: none; background: linear-gradient( to right, var(--thumb-bg) var(--value-percent), var(--track-bg) var(--value-percent) ); border-radius: var(--track-border-radius); } input[type='range']::-moz-range-track { width: 100%; height: var(--track-height); cursor: pointer; background: var(--track-bg); border-radius: var(--track-border-radius); border: none; } input[type='range']::-webkit-slider-thumb { -webkit-appearance: none; appearance: none; height: var(--thumb-size); width: var(--thumb-size); background: var(--thumb-bg); border-radius: var(--thumb-border-radius); box-shadow: var(--thumb-box-shadow); cursor: pointer; margin-top: calc((var(--thumb-size) - var(--track-height)) / -2); } input[type='range']::-moz-range-thumb { height: var(--thumb-size); width: var(--thumb-size); background: var(--thumb-bg); border-radius: var(--thumb-border-radius); box-shadow: var(--thumb-box-shadow); cursor: pointer; border: none; } input[type='number'], input[type='text'], select { background-color: #2a2a2a; color: #eee; border: 1px solid #666; border-radius: 3px; padding: 0.4vmin; font-size: 1.5vmin; font-family: inherit; box-sizing: border-box; } input[type='number'] { width: 6em; } input[type='text'] { width: 100%; } input[type='text']::placeholder { color: #888; } input[type='number']:focus, input[type='text']:focus { outline: none; border-color: #5200ff; box-shadow: 0 0 0 2px rgba(82, 0, 255, 0.3); } select { width: 100%; } select:focus { outline: none; border-color: #5200ff; } select option { background-color: #2a2a2a; color: #eee; } .checkbox-setting { flex-direction: row; align-items: center; gap: 1vmin; } input[type='checkbox'] { cursor: pointer; accent-color: #5200ff; } .core-settings-row { display: flex; flex-direction: row; flex-wrap: wrap; gap: 4vmin; margin-bottom: 1vmin; justify-content: space-evenly; } .core-settings-row .setting { min-width: 16vmin; } .core-settings-row label span:last-child { min-width: 2.5em; } .advanced-toggle { cursor: pointer; margin: 2vmin 0 1vmin 0; color: #aaa; text-decoration: underline; user-select: none; font-size: 1.4vmin; width: fit-content; } .advanced-toggle:hover { color: #eee; } .advanced-settings { display: grid; grid-template-columns: repeat(auto-fit, minmax(10vmin, 1fr)); gap: 3vmin; overflow: hidden; max-height: 0; opacity: 0; transition: max-height 0.3s ease-out, opacity 0.3s ease-out; } .advanced-settings.visible { max-width: 120vmin; max-height: 40vmin; opacity: 1; } hr.divider { display: none; border: none; border-top: 1px solid #666; margin: 2vmin 0; width: 100%; } :host([showadvanced]) hr.divider { display: block; } .auto-row { display: flex; align-items: center; gap: 0.5vmin; } .setting[auto='true'] input[type='range'] { pointer-events: none; filter: grayscale(100%); } .auto-row span { margin-left: auto; } .auto-row label { cursor: pointer; } .auto-row input[type='checkbox'] { cursor: pointer; margin: 0; } `; private readonly defaultConfig: LiveMusicGenerationConfig = { temperature: 1.1, topK: 40, guidance: 4.0, musicGenerationMode: MusicGenerationMode.MUSIC_GENERATION_MODE_QUALITY, }; @state() private config: LiveMusicGenerationConfig = this.defaultConfig; @state() showAdvanced = false; @state() autoDensity = true; @state() lastDefinedDensity: number; @state() autoBrightness = true; @state() lastDefinedBrightness: number; public resetToDefaults() { this.config = this.defaultConfig; this.autoDensity = true; this.lastDefinedDensity = undefined; this.autoBrightness = true; this.lastDefinedBrightness = undefined; this.dispatchSettingsChange(); } private updateSliderBackground(inputEl: HTMLInputElement) { if (inputEl.type !== 'range') { return; } const min = Number(inputEl.min) || 0; const max = Number(inputEl.max) || 100; const value = Number(inputEl.value); const percentage = ((value - min) / (max - min)) * 100; inputEl.style.setProperty('--value-percent', `${percentage}%`); } private handleInputChange(e: Event) { const target = e.target as HTMLInputElement | HTMLSelectElement; const key = target.id as | keyof LiveMusicGenerationConfig | 'auto-density' | 'auto-brightness'; let value: string | number | boolean | undefined = target.value; if (target.type === 'number' || target.type === 'range') { value = target.value === '' ? undefined : Number(target.value); // Update slider background if it's a range input before handling the value change. if (target.type === 'range') { this.updateSliderBackground(target as HTMLInputElement); } } else if (target.type === 'checkbox') { value = (target as HTMLInputElement).checked; } else if (target.type === 'select-one') { const selectElement = target as HTMLSelectElement; if (selectElement.options[selectElement.selectedIndex]?.disabled) { value = undefined; } else { value = target.value; } } const newConfig = { ...this.config, [key]: value, }; if (newConfig.density !== undefined) { this.lastDefinedDensity = newConfig.density; console.log(this.lastDefinedDensity); } if (newConfig.brightness !== undefined) { this.lastDefinedBrightness = newConfig.brightness; } if (key === 'auto-density') { this.autoDensity = Boolean(value); newConfig.density = this.autoDensity ? undefined : this.lastDefinedDensity; } else if (key === 'auto-brightness') { this.autoBrightness = Boolean(value); newConfig.brightness = this.autoBrightness ? undefined : this.lastDefinedBrightness; } this.config = newConfig; this.dispatchSettingsChange(); } override updated(changedProperties: Map) { super.updated(changedProperties); if (changedProperties.has('config')) { this.shadowRoot ?.querySelectorAll('input[type="range"]') .forEach((slider: HTMLInputElement) => { const configValue = this.config[slider.id as keyof LiveMusicGenerationConfig]; if (typeof configValue === 'number') { slider.value = String(configValue); } else if (slider.id === 'density' || slider.id === 'brightness') { // Handle potentially undefined density/brightness with default for background slider.value = String(configValue ?? 0.5); } this.updateSliderBackground(slider); }); } } private dispatchSettingsChange() { this.dispatchEvent( new CustomEvent('settings-changed', { detail: this.config, bubbles: true, composed: true, }), ); } private toggleAdvancedSettings() { this.showAdvanced = !this.showAdvanced; } override render() { const cfg = this.config; const advancedClasses = classMap({ 'advanced-settings': true, 'visible': this.showAdvanced, }); const musicGenerationModeMap = new Map([ ['Quality', MusicGenerationMode.MUSIC_GENERATION_MODE_QUALITY], ['Diversity', MusicGenerationMode.MUSIC_GENERATION_MODE_DIVERSITY], ['Vocalization', MusicGenerationMode.MUSIC_GENERATION_MODE_VOCALIZATION], ]); const scaleMap = new Map([ ['Auto', 'SCALE_UNSPECIFIED'], ['C Major / A Minor', 'C_MAJOR_A_MINOR'], ['C# Major / A# Minor', 'D_FLAT_MAJOR_B_FLAT_MINOR'], ['D Major / B Minor', 'D_MAJOR_B_MINOR'], ['D# Major / C Minor', 'E_FLAT_MAJOR_C_MINOR'], ['E Major / C# Minor', 'E_MAJOR_D_FLAT_MINOR'], ['F Major / D Minor', 'F_MAJOR_D_MINOR'], ['F# Major / D# Minor', 'G_FLAT_MAJOR_E_FLAT_MINOR'], ['G Major / E Minor', 'G_MAJOR_E_MINOR'], ['G# Major / F Minor', 'A_FLAT_MAJOR_F_MINOR'], ['A Major / F# Minor', 'A_MAJOR_G_FLAT_MINOR'], ['A# Major / G Minor', 'B_FLAT_MAJOR_G_MINOR'], ['B Major / G# Minor', 'B_MAJOR_A_FLAT_MINOR'], ]); return html`

${(this.lastDefinedDensity ?? 0.5).toFixed(2)}
${(this.lastDefinedBrightness ?? 0.5).toFixed(2)}
${this.showAdvanced ? 'Hide' : 'Show'} Advanced Settings
`; } } /** Component for the PromptDJ UI. */ @customElement('prompt-dj') class PromptDj extends LitElement { static override styles = css` :host { height: 100%; width: 100%; display: flex; flex-direction: column; justify-content: center; align-items: center; box-sizing: border-box; padding: 2vmin; position: relative; font-size: 1.8vmin; } #background { position: absolute; height: 100%; width: 100%; z-index: -1; background: #111; } .prompts-area { display: flex; align-items: flex-end; justify-content: center; flex: 4; width: 100%; margin-top: 2vmin; gap: 2vmin; } #prompts-container { display: flex; flex-direction: row; align-items: flex-end; flex-shrink: 1; height: 100%; gap: 2vmin; margin-left: 10vmin; padding: 1vmin; overflow-x: auto; scrollbar-width: thin; scrollbar-color: #666 #1a1a1a; } #prompts-container::-webkit-scrollbar { height: 8px; } #prompts-container::-webkit-scrollbar-track { background: #111; border-radius: 4px; } #prompts-container::-webkit-scrollbar-thumb { background-color: #666; border-radius: 4px; } #prompts-container::-webkit-scrollbar-thumb:hover { background-color: #777; } /* Add pseudo-elements for centering while keeping elements visible when scrolling */ #prompts-container::before, #prompts-container::after { content: ''; flex: 1; min-width: 0.5vmin; } .add-prompt-button-container { display: flex; align-items: flex-end; height: 100%; flex-shrink: 0; } #settings-container { flex: 1; margin: 2vmin 0 1vmin 0; } .playback-container { display: flex; justify-content: center; align-items: center; flex-shrink: 0; } play-pause-button, add-prompt-button, reset-button { width: 12vmin; flex-shrink: 0; } prompt-controller { height: 100%; max-height: 80vmin; min-width: 14vmin; max-width: 16vmin; flex: 1; } `; @property({ type: Object, attribute: false, }) private prompts: Map; private nextPromptId: number; // Monotonically increasing ID for new prompts private session: LiveMusicSession; private readonly sampleRate = 48000; private audioContext: AudioContext; private outputNode: GainNode; private nextStartTime = 0; private readonly bufferTime = 2; // adds an audio buffer in case of netowrk latency @state() private playbackState: PlaybackState = 'stopped'; @property({type: Object}) private filteredPrompts = new Set(); private connectionError = true; @query('play-pause-button') private playPauseButton!: PlayPauseButton; @query('toast-message') private toastMessage!: ToastMessage; @query('settings-controller') private settingsController!: SettingsController; constructor(prompts: Map) { super(); this.prompts = prompts; this.nextPromptId = this.prompts.size; this.audioContext = new (window.AudioContext || (window as any).webkitAudioContext)({sampleRate: this.sampleRate}); this.outputNode = this.audioContext.createGain(); this.outputNode.connect(this.audioContext.destination); } override async firstUpdated() { await this.connectToSession(); this.setSessionPrompts(); } private async connectToSession() { this.session = await ai.live.music.connect({ model: model, callbacks: { onmessage: async (e: LiveMusicServerMessage) => { console.log('Received message from the server: %s\n'); console.log(e); if (e.setupComplete) { this.connectionError = false; } if (e.filteredPrompt) { this.filteredPrompts = new Set([ ...this.filteredPrompts, e.filteredPrompt.text, ]); this.toastMessage.show(e.filteredPrompt.filteredReason); } if (e.serverContent?.audioChunks?.length) { if ( this.playbackState === 'paused' || this.playbackState === 'stopped' ) return; const audioBuffer = await decodeAudioData( decode(e.serverContent.audioChunks[0].data), this.audioContext, 48000, 2, ); const source = this.audioContext.createBufferSource(); source.buffer = audioBuffer; source.connect(this.outputNode); if (this.nextStartTime === 0) { this.nextStartTime = this.audioContext.currentTime + this.bufferTime; setTimeout(() => { this.playbackState = 'playing'; }, this.bufferTime * 1000); } if (this.nextStartTime < this.audioContext.currentTime) { console.log('under run'); this.playbackState = 'loading'; this.nextStartTime = 0; return; } source.start(this.nextStartTime); this.nextStartTime += audioBuffer.duration; } }, onerror: (e: ErrorEvent) => { console.log('Error occurred: %s\n', JSON.stringify(e)); this.connectionError = true; this.stopAudio(); this.toastMessage.show('Connection error, please restart audio.'); }, onclose: (e: CloseEvent) => { console.log('Connection closed.'); this.connectionError = true; this.stopAudio(); this.toastMessage.show('Connection error, please restart audio.'); }, }, }); } private setSessionPrompts = throttle(async () => { const promptsToSend = Array.from(this.prompts.values()).filter((p) => { return !this.filteredPrompts.has(p.text) && p.weight !== 0; }); try { await this.session.setWeightedPrompts({ weightedPrompts: promptsToSend, }); } catch (e) { this.toastMessage.show(e.message); this.pauseAudio(); } }, 200); private dispatchPromptsChange() { this.dispatchEvent( new CustomEvent('prompts-changed', {detail: this.prompts}), ); } private handlePromptChanged(e: CustomEvent) { const {promptId, text, weight} = e.detail; const prompt = this.prompts.get(promptId); if (!prompt) { console.error('prompt not found', promptId); return; } prompt.text = text; prompt.weight = weight; const newPrompts = new Map(this.prompts); newPrompts.set(promptId, prompt); this.prompts = newPrompts; this.setSessionPrompts(); this.requestUpdate(); this.dispatchPromptsChange(); } /** Generates radial gradients for each prompt based on weight and color. */ private makeBackground() { const clamp01 = (v: number) => Math.min(Math.max(v, 0), 1); const MAX_WEIGHT = 0.5; const MAX_ALPHA = 0.6; const bg: string[] = []; [...this.prompts.values()].forEach((p, i) => { const alphaPct = clamp01(p.weight / MAX_WEIGHT) * MAX_ALPHA; const alpha = Math.round(alphaPct * 0xff) .toString(16) .padStart(2, '0'); const stop = p.weight / 2; const x = (i % 4) / 3; const y = Math.floor(i / 4) / 3; const s = `radial-gradient(circle at ${x * 100}% ${y * 100}%, ${p.color}${alpha} 0px, ${p.color}00 ${stop * 100}%)`; bg.push(s); }); return bg.join(', '); } private async handlePlayPause() { if (this.playbackState === 'playing') { this.pauseAudio(); } else if ( this.playbackState === 'paused' || this.playbackState === 'stopped' ) { if (this.connectionError) { await this.connectToSession(); this.setSessionPrompts(); } this.loadAudio(); } else if (this.playbackState === 'loading') { this.stopAudio(); } console.debug('handlePlayPause'); } private pauseAudio() { this.session.pause(); this.playbackState = 'paused'; this.outputNode.gain.setValueAtTime(1, this.audioContext.currentTime); this.outputNode.gain.linearRampToValueAtTime( 0, this.audioContext.currentTime + 0.1, ); this.nextStartTime = 0; this.outputNode = this.audioContext.createGain(); this.outputNode.connect(this.audioContext.destination); } private loadAudio() { this.audioContext.resume(); this.session.play(); this.playbackState = 'loading'; this.outputNode.gain.setValueAtTime(0, this.audioContext.currentTime); this.outputNode.gain.linearRampToValueAtTime( 1, this.audioContext.currentTime + 0.1, ); } private stopAudio() { this.session.stop(); this.playbackState = 'stopped'; this.outputNode.gain.setValueAtTime(0, this.audioContext.currentTime); this.outputNode.gain.linearRampToValueAtTime( 1, this.audioContext.currentTime + 0.1, ); this.nextStartTime = 0; } private async handleAddPrompt() { const newPromptId = `prompt-${this.nextPromptId++}`; const usedColors = [...this.prompts.values()].map((p) => p.color); const newPrompt: Prompt = { promptId: newPromptId, text: 'New Prompt', // Default text weight: 0, color: getUnusedRandomColor(usedColors), }; const newPrompts = new Map(this.prompts); newPrompts.set(newPromptId, newPrompt); this.prompts = newPrompts; await this.setSessionPrompts(); // Wait for the component to update and render the new prompt. // Do not dispatch the prompt change event until the user has edited the prompt text. await this.updateComplete; // Find the newly added prompt controller element const newPromptElement = this.renderRoot.querySelector( `prompt-controller[promptId="${newPromptId}"]`, ); if (newPromptElement) { // Scroll the prompts container to the new prompt element newPromptElement.scrollIntoView({ behavior: 'smooth', block: 'nearest', inline: 'end', }); // Select the new prompt text const textSpan = newPromptElement.shadowRoot?.querySelector('#text'); if (textSpan) { textSpan.focus(); const selection = window.getSelection(); const range = document.createRange(); range.selectNodeContents(textSpan); selection?.removeAllRanges(); selection?.addRange(range); } } } private handlePromptRemoved(e: CustomEvent) { e.stopPropagation(); const promptIdToRemove = e.detail; if (this.prompts.has(promptIdToRemove)) { this.prompts.delete(promptIdToRemove); const newPrompts = new Map(this.prompts); this.prompts = newPrompts; this.setSessionPrompts(); this.dispatchPromptsChange(); } else { console.warn( `Attempted to remove non-existent prompt ID: ${promptIdToRemove}`, ); } } // Handle scrolling X-axis the prompts container. private handlePromptsContainerWheel(e: WheelEvent) { const container = e.currentTarget as HTMLElement; if (e.deltaX !== 0) { // Prevent the default browser action (like page back/forward) e.preventDefault(); container.scrollLeft += e.deltaX; } } private updateSettings = throttle( async (e: CustomEvent) => { await this.session?.setMusicGenerationConfig({ musicGenerationConfig: e.detail, }); }, 200, ); private async handleReset() { if (this.connectionError) { await this.connectToSession(); this.setSessionPrompts(); } this.pauseAudio(); this.session.resetContext(); this.settingsController.resetToDefaults(); this.session?.setMusicGenerationConfig({ musicGenerationConfig: {}, }); setTimeout(this.loadAudio.bind(this), 100); } override render() { const bg = styleMap({ backgroundImage: this.makeBackground(), }); return html`
${this.renderPrompts()}
`; } private renderPrompts() { return [...this.prompts.values()].map((prompt) => { return html` `; }); } } function gen(parent: HTMLElement) { const initialPrompts = getStoredPrompts(); const pdj = new PromptDj(initialPrompts); parent.appendChild(pdj); } function getStoredPrompts(): Map { const {localStorage} = window; const storedPrompts = localStorage.getItem('prompts'); if (storedPrompts) { try { const prompts = JSON.parse(storedPrompts) as Prompt[]; console.log('Loading stored prompts', prompts); return new Map(prompts.map((prompt) => [prompt.promptId, prompt])); } catch (e) { console.error('Failed to parse stored prompts', e); } } console.log('No stored prompts, creating prompt presets'); const numDefaultPrompts = Math.min(4, PROMPT_TEXT_PRESETS.length); const shuffledPresetTexts = [...PROMPT_TEXT_PRESETS].sort( () => Math.random() - 0.5, ); const defaultPrompts: Prompt[] = []; const usedColors: string[] = []; for (let i = 0; i < numDefaultPrompts; i++) { const text = shuffledPresetTexts[i]; const color = getUnusedRandomColor(usedColors); usedColors.push(color); defaultPrompts.push({ promptId: `prompt-${i}`, text, weight: 0, color, }); } // Randomly select up to 2 prompts to set their weight to 1. const promptsToActivate = [...defaultPrompts].sort(() => Math.random() - 0.5); const numToActivate = Math.min(2, defaultPrompts.length); for (let i = 0; i < numToActivate; i++) { if (promptsToActivate[i]) { promptsToActivate[i].weight = 1; } } return new Map(defaultPrompts.map((p) => [p.promptId, p])); } function setStoredPrompts(prompts: Map) { const storedPrompts = JSON.stringify([...prompts.values()]); const {localStorage} = window; localStorage.setItem('prompts', storedPrompts); } function main(container: HTMLElement) { gen(container); } main(document.body); declare global { interface HTMLElementTagNameMap { 'prompt-dj': PromptDj; 'prompt-controller': PromptController; 'settings-controller': SettingsController; 'add-prompt-button': AddPromptButton; 'play-pause-button': PlayPauseButton; 'reset-button': ResetButton; 'weight-slider': WeightSlider; 'toast-message': ToastMessage; } }