adding latest working version

This commit is contained in:
2020-03-07 16:59:11 +00:00
parent eed16b9128
commit 3d1aeaed3c
20 changed files with 33736 additions and 60062 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

View File

@@ -1,189 +1,189 @@
{
"Please Please Me [1963]": {
"avg": 150,
"median": 151,
"std": 38,
"max": 208,
"min": 92,
"p_10": 104,
"avg": 158,
"median": 160,
"std": 53,
"max": 288,
"min": 103,
"p_10": 106,
"p_25": 108,
"p_75": 185,
"p_90": 186,
"count": 13
"p_90": 200,
"count": 12
},
"With the Beatles [1963]": {
"avg": 143,
"median": 156,
"std": 51,
"max": 227,
"min": 54,
"p_10": 64,
"p_25": 106,
"p_75": 172,
"p_90": 194,
"avg": 174,
"median": 170,
"std": 60,
"max": 274,
"min": 59,
"p_10": 92,
"p_25": 143,
"p_75": 215,
"p_90": 241,
"count": 14
},
"Introducing\u2026 The Beatles [1964]": {
"avg": 143,
"median": 143,
"std": 38,
"max": 208,
"min": 92,
"p_10": 103,
"p_25": 108,
"p_75": 176,
"p_90": 186,
"avg": 167,
"median": 151,
"std": 59,
"max": 288,
"min": 105,
"p_10": 108,
"p_25": 114,
"p_75": 193,
"p_90": 257,
"count": 11
},
"Meet the Beatles! [2014]": {
"avg": 142,
"median": 151,
"std": 49,
"avg": 160,
"median": 158,
"std": 33,
"max": 215,
"min": 54,
"p_10": 64,
"p_25": 94,
"p_75": 181,
"p_90": 196,
"min": 91,
"p_10": 130,
"p_25": 141,
"p_75": 179,
"p_90": 200,
"count": 24
},
"Twist and Shout [1964]": {
"avg": 178,
"median": 173,
"std": 55,
"max": 288,
"avg": 170,
"median": 176,
"std": 44,
"max": 257,
"min": 105,
"p_10": 109,
"p_25": 141,
"p_75": 205,
"p_90": 253,
"p_75": 198,
"p_90": 204,
"count": 12
},
"The Beatles\u2019 Second Album [2004]": {
"avg": 192,
"median": 184,
"std": 69,
"avg": 205,
"median": 199,
"std": 59,
"max": 339,
"min": 82,
"p_10": 114,
"min": 135,
"p_10": 139,
"p_25": 155,
"p_75": 227,
"p_90": 274,
"count": 21
"p_75": 240,
"p_90": 251,
"count": 20
},
"The Beatles\u2019 Long Tall Sally [1964]": {
"avg": 171,
"median": 171,
"std": 46,
"max": 241,
"min": 81,
"p_10": 131,
"p_25": 138,
"p_75": 193,
"p_90": 238,
"avg": 182,
"median": 180,
"std": 70,
"max": 339,
"min": 82,
"p_10": 94,
"p_25": 134,
"p_75": 221,
"p_90": 241,
"count": 12
},
"Something New [2014]": {
"avg": 142,
"median": 147,
"std": 36,
"max": 185,
"min": 77,
"p_10": 82,
"p_25": 118,
"p_75": 170,
"p_90": 180,
"avg": 172,
"median": 178,
"std": 28,
"max": 220,
"min": 103,
"p_10": 146,
"p_25": 165,
"p_75": 185,
"p_90": 187,
"count": 22
},
"Beatles for Sale [1964]": {
"avg": 140,
"median": 127,
"std": 68,
"max": 316,
"avg": 147,
"median": 146,
"std": 76,
"max": 358,
"min": 53,
"p_10": 74,
"p_25": 102,
"p_75": 158,
"p_75": 159,
"p_90": 202,
"count": 12
},
"Beatles \u201965 [2004]": {
"avg": 155,
"median": 146,
"std": 64,
"avg": 161,
"median": 147,
"std": 69,
"max": 358,
"min": 71,
"p_10": 104,
"p_25": 117,
"p_25": 141,
"p_75": 155,
"p_90": 171,
"p_90": 179,
"count": 21
},
"Beatles VI [2014]": {
"avg": 142,
"median": 140,
"std": 46,
"max": 205,
"avg": 159,
"median": 177,
"std": 52,
"max": 220,
"min": 53,
"p_10": 84,
"p_25": 110,
"p_75": 173,
"p_90": 194,
"p_25": 137,
"p_75": 190,
"p_90": 210,
"count": 18
},
"Rubber Soul [1999]": {
"avg": 155,
"median": 144,
"std": 40,
"avg": 175,
"median": 168,
"std": 42,
"max": 239,
"min": 89,
"p_10": 115,
"p_25": 127,
"p_75": 169,
"p_90": 223,
"count": 22
},
"Revolver [2014]": {
"avg": 144,
"median": 134,
"std": 41,
"max": 212,
"min": 90,
"p_10": 92,
"p_25": 105,
"p_75": 165,
"p_90": 205,
"count": 22
},
"Sgt. Pepper\u2019s Lonely Hearts Club Band [1994]": {
"avg": 177,
"median": 169,
"std": 48,
"max": 297,
"min": 87,
"p_10": 131,
"p_25": 150,
"p_75": 198,
"p_10": 133,
"p_25": 147,
"p_75": 217,
"p_90": 227,
"count": 20
},
"Revolver [2014]": {
"avg": 159,
"median": 160,
"std": 48,
"max": 233,
"min": 90,
"p_10": 102,
"p_25": 113,
"p_75": 205,
"p_90": 214,
"count": 22
},
"Sgt. Pepper\u2019s Lonely Hearts Club Band [1994]": {
"avg": 194,
"median": 176,
"std": 69,
"max": 395,
"min": 87,
"p_10": 136,
"p_25": 151,
"p_75": 217,
"p_90": 276,
"count": 19
},
"The Beatles [1968]": {
"avg": 124,
"median": 124,
"std": 68,
"max": 272,
"avg": 139,
"median": 131,
"std": 91,
"max": 324,
"min": 9,
"p_10": 26,
"p_25": 111,
"p_75": 146,
"p_90": 205,
"p_25": 72,
"p_75": 198,
"p_90": 255,
"count": 15
},
"Abbey Road [2009]": {
"avg": 84,
"median": 72,
"std": 43,
"max": 154,
"avg": 91,
"median": 76,
"std": 55,
"max": 234,
"min": 28,
"p_10": 35,
"p_25": 52,
@@ -192,40 +192,40 @@
"count": 14
},
"Let It Be [2010]": {
"avg": 162,
"median": 154,
"std": 62,
"max": 244,
"avg": 173,
"median": 177,
"std": 70,
"max": 268,
"min": 57,
"p_10": 69,
"p_25": 117,
"p_75": 204,
"p_90": 243,
"p_75": 230,
"p_90": 264,
"count": 13
},
"Decca Sessions 1.1.62 [1987]": {
"avg": 182,
"median": 167,
"std": 56,
"std": 55,
"max": 289,
"min": 107,
"p_10": 126,
"p_25": 130,
"p_75": 227,
"p_75": 223,
"p_90": 241,
"count": 9
},
"The Unreleased Tracks, Volume 2 [1990]": {
"avg": 142,
"median": 125,
"std": 43,
"avg": 141,
"median": 119,
"std": 45,
"max": 241,
"min": 105,
"p_10": 108,
"p_25": 110,
"p_75": 143,
"p_90": 195,
"count": 9
"p_75": 153,
"p_90": 201,
"count": 8
},
"Reunions 74 & 92 [1992]": {
"avg": 149,
@@ -240,60 +240,60 @@
"count": 6
},
"Studio 2 Sessions at Abbey Road, Vol. 1 [1995]": {
"avg": 133,
"median": 112,
"std": 35,
"avg": 132,
"median": 108,
"std": 36,
"max": 185,
"min": 92,
"min": 105,
"p_10": 105,
"p_25": 105,
"p_75": 175,
"p_75": 185,
"p_90": 185,
"count": 22
"count": 20
},
"Studio 2 Sessions at Abbey Road, Vol. 2 [1995]": null,
"Studio 2 Sessions at Abbey Road, Vol. 3 [1995]": {
"avg": 177,
"median": 147,
"std": 52,
"avg": 179,
"median": 163,
"std": 51,
"max": 248,
"min": 117,
"p_10": 117,
"p_25": 142,
"p_25": 146,
"p_75": 248,
"p_90": 248,
"count": 16
},
"Studio 2 Sessions at Abbey Road, Vol. 4 [1995]": {
"avg": 199,
"median": 207,
"std": 53,
"avg": 222,
"median": 233,
"std": 46,
"max": 269,
"min": 127,
"p_10": 127,
"p_25": 148,
"p_75": 237,
"min": 145,
"p_10": 150,
"p_25": 188,
"p_75": 269,
"p_90": 269,
"count": 18
},
"The Alternate Abbey Road [2000]": {
"avg": 91,
"median": 81,
"std": 42,
"max": 154,
"min": 28,
"p_10": 36,
"avg": 100,
"median": 76,
"std": 59,
"max": 234,
"min": 32,
"p_10": 37,
"p_25": 57,
"p_75": 128,
"p_90": 137,
"p_75": 146,
"p_90": 178,
"count": 16
},
"Alf Together Now [2001]": {
"avg": 192,
"avg": 193,
"median": 170,
"std": 68,
"max": 339,
"min": 108,
"min": 111,
"p_10": 164,
"p_25": 168,
"p_75": 170,
@@ -301,73 +301,73 @@
"count": 12
},
"Collectors Items [2002]": {
"avg": 207,
"median": 192,
"std": 126,
"avg": 241,
"median": 204,
"std": 160,
"max": 639,
"min": 89,
"p_10": 100,
"p_25": 139,
"p_75": 231,
"p_90": 267,
"p_10": 101,
"p_25": 154,
"p_75": 243,
"p_90": 457,
"count": 16
},
"Complete Home Recordings 1967\u20131968 [2002]": {
"avg": 181,
"median": 199,
"std": 71,
"max": 274,
"min": 32,
"p_10": 107,
"p_25": 113,
"p_75": 229,
"avg": 197,
"median": 204,
"std": 74,
"max": 346,
"min": 70,
"p_10": 111,
"p_25": 111,
"p_75": 270,
"p_90": 270,
"count": 20
},
"Complete Home Recordings 1968 [2002]": {
"avg": 150,
"median": 136,
"std": 82,
"avg": 160,
"median": 153,
"std": 88,
"max": 351,
"min": 32,
"p_10": 63,
"p_25": 107,
"p_75": 163,
"p_10": 41,
"p_25": 94,
"p_75": 209,
"p_90": 272,
"count": 24
},
"The Seven Years of Christmas [2002]": {
"avg": 695,
"avg": 790,
"median": 659,
"std": 339,
"std": 202,
"max": 1122,
"min": 114,
"p_10": 332,
"min": 589,
"p_10": 617,
"p_25": 659,
"p_75": 921,
"p_90": 1042,
"count": 5
},
"20 X 4 Remastered Edition [2005]": {
"avg": 158,
"median": 145,
"std": 61,
"max": 230,
"avg": 176,
"median": 178,
"std": 69,
"max": 269,
"min": 66,
"p_10": 91,
"p_25": 128,
"p_10": 98,
"p_25": 145,
"p_75": 220,
"p_90": 226,
"p_90": 250,
"count": 5
},
"Inside Sgt. Pepper (Part Three) [2007]": {
"avg": 228,
"avg": 235,
"median": 235,
"std": 11,
"std": 0,
"max": 235,
"min": 212,
"p_10": 212,
"p_25": 212,
"min": 235,
"p_10": 235,
"p_25": 235,
"p_75": 235,
"p_90": 235,
"count": 9
@@ -398,63 +398,63 @@
"count": 4
},
"From the Basement to the Boardroom [2011]": {
"avg": 125,
"median": 121,
"std": 69,
"max": 227,
"avg": 179,
"median": 173,
"std": 161,
"max": 589,
"min": 32,
"p_10": 32,
"p_25": 54,
"p_75": 173,
"p_90": 203,
"p_75": 194,
"p_90": 297,
"count": 9
},
"The Decca Tapes [2013]": {
"avg": 163,
"median": 165,
"std": 60,
"avg": 172,
"median": 167,
"std": 56,
"max": 289,
"min": 94,
"p_10": 97,
"p_25": 107,
"p_75": 206,
"p_90": 228,
"count": 13
"min": 101,
"p_10": 108,
"p_25": 127,
"p_75": 211,
"p_90": 229,
"count": 12
},
"Kinfauns Demos [Missing]": {
"avg": 144,
"median": 142,
"std": 71,
"max": 299,
"avg": 153,
"median": 150,
"std": 84,
"max": 324,
"min": 32,
"p_10": 51,
"p_25": 94,
"p_75": 198,
"p_90": 230,
"p_10": 32,
"p_25": 82,
"p_75": 208,
"p_90": 271,
"count": 28
},
"Rare Tracks [Missing]": {
"avg": 181,
"median": 186,
"std": 28,
"max": 212,
"min": 126,
"p_10": 148,
"p_25": 168,
"p_75": 204,
"p_90": 210,
"avg": 235,
"median": 222,
"std": 67,
"max": 395,
"min": 171,
"p_10": 176,
"p_25": 190,
"p_75": 244,
"p_90": 299,
"count": 8
},
"The Lost Album (Two and a Half) [Missing]": {
"avg": 169,
"median": 161,
"std": 61,
"avg": 168,
"median": 163,
"std": 67,
"max": 282,
"min": 32,
"p_10": 111,
"p_25": 123,
"p_75": 203,
"p_90": 243,
"p_10": 112,
"p_25": 122,
"p_75": 210,
"p_90": 265,
"count": 22
},
"8 Mile and Abbey: Eminem Meets the Beatles [2014]": null

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

BIN
src/.DS_Store vendored Normal file

Binary file not shown.

BIN
src/musicbrainzapi/.DS_Store vendored Normal file

Binary file not shown.

BIN
src/musicbrainzapi/api/.DS_Store vendored Normal file

Binary file not shown.

Binary file not shown.

View File

@@ -383,9 +383,9 @@ class LyricsBuilder(LyricsConcreteBuilder):
# rename this
def calculate_average_all_albums(self) -> None:
self.all_albums_lyrics_sum = list()
album_lyrics = self.all_albums_lyrics_count
# with open(f'{os.getcwd()}/lyrics_count.json', 'r') as f:
# album_lyrics = json.load(f)
# album_lyrics = self.all_albums_lyrics_count
with open(f'{os.getcwd()}/lyrics_count.json', 'r') as f:
album_lyrics = json.load(f)
count = 0
for i in album_lyrics:
count += len(i)
@@ -438,9 +438,9 @@ class LyricsBuilder(LyricsConcreteBuilder):
def calculate_final_average_by_year(self) -> None:
group_by_years = addict.Dict()
self.year_statistics = addict.Dict()
album_lyrics = self.all_albums_lyrics_sum
# with open(f'{os.getcwd()}/lyrics_sum_all_album.json', 'r') as f:
# album_lyrics = json.load(f)
# album_lyrics = self.all_albums_lyrics_sum
with open(f'{os.getcwd()}/lyrics_sum_all_album.json', 'r') as f:
album_lyrics = json.load(f)
# Merge years together
for i in album_lyrics:
@@ -735,6 +735,7 @@ class Lyrics:
stats[9],
]
for group, s in stats_obj.items():
try:
output_1.append_row(
[
group,
@@ -750,6 +751,8 @@ class Lyrics:
s.get(stats[9]),
]
)
except AttributeError:
continue
output_0.append_row([output_1])
click.echo(output_0)
return self

File diff suppressed because it is too large Load Diff

View File

View File

@@ -33,12 +33,7 @@ class ComplexCLI(click.MultiCommand):
def get_command(self, ctx, name):
try:
if sys.version_info[0] == 2:
name = name.encode('ascii', 'replace')
mod = import_module(f'musicbrainzapi.cli.commands.cmd_{name}')
# mod = __import__(
# 'complex.commands.cmd_' + name, None, None, ['cli']
# )
except ImportError as e:
print(e)
return
@@ -50,23 +45,23 @@ class ComplexCLI(click.MultiCommand):
'-p',
'--path',
type=click.Path(
exists=False, file_okay=False, resolve_path=True, writable=True
exists=True, file_okay=False, resolve_path=True, writable=True
),
help='Path to save results.',
default=os.path.expanduser('~/.musicbrainzapi')
default=os.getcwd()
)
@click.option('-v', '--verbose', is_flag=True, help='Enables verbose mode.')
# @click.option('-v', '--verbose', is_flag=True, help='Enables verbose mode.')
@click.version_option(
version=__version__,
prog_name=__header__,
message=f'{__header__} version {__version__} 🎤',
)
@pass_environment
def cli(ctx, verbose, path):
def cli(ctx, path):
"""A complex command line interface."""
ctx.verbose = verbose
# ctx.verbose = verbose
if path is not None:
click.echo(f'Path set to {os.path.expanduser(path)}')
# click.echo(f'Path set to {os.path.expanduser(path)}')
ctx.path = os.path.expanduser(path)

View File

@@ -2,8 +2,11 @@ from typing import Union
import click
import matplotlib.pyplot as plt
from musicbrainzapi.cli.cli import pass_environment
from musicbrainzapi.api.command_builders import lyrics
import musicbrainzapi.wordcloud
# @click.argument('path', required=False, type=click.Path(resolve_path=True))
@@ -17,7 +20,14 @@ from musicbrainzapi.api.command_builders import lyrics
help='Save the output to json files locally. Will use the path parameter if'
' provided else defaults to current working directory.',
is_flag=True,
default=False
default=False,
)
@click.option(
'--wordcloud',
required=False,
help='Generate a wordcloud from lyrics.',
is_flag=True,
default=False,
)
@click.option(
'--show-summary',
@@ -51,9 +61,12 @@ def cli(
country: Union[str, None],
dev: bool,
show_summary: str,
save_output: bool
wordcloud: bool,
save_output: bool,
) -> None:
"""Search for lyrics statistics of an Artist/Group."""
path = ctx.path
print(f'home={ctx.home}')
# lyrics_obj = list()
director = lyrics.LyricsClickDirector()
builder = lyrics.LyricsBuilder()
@@ -77,9 +90,27 @@ def cli(
# Show basic count
lyrics_0.show_summary()
# Show summary statistics
if show_summary == 'all':
lyrics_0.show_summary_statistics(group_by='album')
lyrics_0.show_summary_statistics(group_by='year')
elif show_summary in ['album', 'year']:
lyrics_0.show_summary_statistics(group_by=show_summary)
# Show wordcloud
if wordcloud:
click.echo('Generating wordcloud')
cloud = musicbrainzapi.wordcloud.LyricsWordcloud.use_microphone(
lyrics_0.all_albums_lyrics_count
)
cloud.create_word_cloud()
click.confirm('Wordcloud ready - press enter to show.', default=True)
plt.imshow(
cloud.wc.recolor(
color_func=cloud.generate_grey_colours, random_state=3
),
interpolation='bilinear',
)
plt.axis('off')
plt.show()

View File

@@ -0,0 +1,115 @@
from __future__ import annotations
import collections
from importlib import resources
import random
import re
import typing
from matplotlib import pyplot as plt
from PIL import Image
from wordcloud import STOPWORDS, WordCloud
import numpy as np
from musicbrainzapi.api.command_builders.lyrics import Lyrics
if typing.TYPE_CHECKING:
import PIL.PngImagePlugin.PngImageFile
class LyricsWordcloud:
"""docstring for LyricsWordcloud"""
def __init__(
self,
pillow_img: 'PIL.PngImagePlugin.PngImageFile',
all_albums_lyrics_count: 'Lyrics.all_albums_lyrics_count',
):
self.pillow_img = pillow_img
self.all_albums_lyrics_count = all_albums_lyrics_count
@classmethod
def use_microphone(
cls, all_albums_lyrics_count: 'Lyrics.all_albums_lyrics_count',
) -> LyricsWordcloud:
mic_resource = resources.path(
'musicbrainzapi.wordcloud.resources', 'mic.png'
)
with mic_resource as m:
mic_img = Image.open(m)
return cls(mic_img, all_albums_lyrics_count)
def _get_lyrics_list(self) -> None:
self.lyrics_list = list()
for i in self.all_albums_lyrics_count:
for album, lyric in i.items():
for track in lyric:
try:
for word in track:
for _ in range(1, word[1]):
cleaned = word[0]
cleaned = re.sub(
r'[\(\[].*?[\)\]]', ' ', cleaned
)
cleaned = re.sub(
r'[^a-zA-Z0-9\s]', '', cleaned
)
cleaned = cleaned.lower()
if cleaned in STOPWORDS:
continue
self.lyrics_list.append(cleaned)
except IndexError:
pass
return self
def _get_frequencies(self) -> None:
self.freq = collections.Counter(self.lyrics_list)
def _get_char_mask(self) -> None:
self.char_mask = np.array(self.pillow_img)
@staticmethod
def generate_grey_colours(
word: str,
font_size: str,
random_state: typing.Union[None, bool] = None,
*args,
**kwargs,
) -> str:
colour = f'hsl(0, 0%, {random.randint(60, 100)}%)'
return colour
def _generate_word_cloud(self) -> None:
self.wc = WordCloud(
max_words=50,
width=500,
height=500,
mask=self.char_mask,
random_state=1,
).generate_from_frequencies(self.freq)
return self
def _generate_plot(self) -> None:
plt.imshow(
self.wc.recolor(
color_func=self.generate_grey_colours, random_state=3
),
interpolation='bilinear',
)
plt.axis('off')
return self
def save_to_disk(self, path: str):
pass
def show_word_cloud(self):
plt.show()
def create_word_cloud(self) -> None:
self._get_lyrics_list()
self._get_frequencies()
self._get_char_mask()
self._generate_word_cloud()
self._generate_plot()
return self

Binary file not shown.

After

Width:  |  Height:  |  Size: 186 KiB