Skip to content

Commit 0e1af77

Browse files
vitlindanicolasfara
vitlinda
authored and
nicolasfara
committed
feat: table generation from all entities file
1 parent ebebe22 commit 0e1af77

9 files changed

+155
-151
lines changed

‎src/main/scala/dev/atedeg/Configuration.scala

+9-62
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,5 @@
11
package dev.atedeg
22

3-
import scala.util.Try
4-
5-
import better.files.File
6-
import io.circe.parser.{ parse => parseJsonString }
7-
import io.circe.{ Decoder, Json }
8-
import io.circe.yaml.parser
9-
import cats.implicits._
10-
import Extensions._
11-
123
sealed trait EntityType {
134
override def toString: String = this match {
145
case Class => "class"
@@ -39,61 +30,17 @@ case object Type extends EntityType
3930
case object Case extends EntityType
4031
case object Def extends EntityType
4132

42-
final case class Entity(entityType: EntityType, link: String, name: String, packageName: String)
43-
final case class BaseEntity(entityType: EntityType, name: String)
44-
45-
object Entity {
46-
private val entitiesFileName = "searchData.js"
47-
48-
private def allEntitiesFile(workingDir: File): File =
49-
workingDir / "target" / "site" / "scripts" / entitiesFileName
50-
51-
def readAll(workingDir: File): Either[Error, Set[Entity]] = Utils.parseFileWith(allEntitiesFile(workingDir))(parse)
52-
53-
private[atedeg] def parse(raw: String): Either[Error, Set[Entity]] =
54-
parseJsonString(raw).leftMap(CirceParsingFailure).flatMap(parseJson)
55-
56-
private def parseJson(json: Json): Either[Error, Set[Entity]] = {
57-
def build(maybeType: Option[EntityType], link: String, name: String, packageName: String): Option[Entity] =
58-
maybeType.map(Entity(_, link, name, packageName))
59-
60-
implicit val decodeEntityType: Decoder[Option[EntityType]] = Decoder.decodeString.map(EntityType.fromString)
61-
implicit val decodeEntity: Decoder[Option[Entity]] = Decoder.forProduct4("k", "l", "n", "d")(build)
62-
json.as[Set[Option[Entity]]].map(_.dropNone).leftMap(CirceDecodingFailure)
33+
final case class Entity(entityType: EntityType, link: String, name: String, packageName: String) {
34+
def toBaseEntity: BaseEntity = BaseEntity(entityType, name)
35+
def sanitizedLink: String = link.split('#').head
36+
def entityId: Option[String] = link.split('#').lastOption
37+
def isClassLike: Boolean = entityType match {
38+
case Class | Trait | Enum => true
39+
case Type | Case | Def => false
6340
}
6441
}
65-
42+
final case class BaseEntity(entityType: EntityType, name: String)
6643
final case class Configuration(ignored: Set[BaseEntity], tables: List[TableConfig])
67-
final case class TableConfig(
68-
name: String,
69-
termName: Option[String],
70-
definitionName: Option[String],
71-
rows: List[BaseEntity],
72-
)
73-
74-
object Configuration {
75-
private val configFile = ".ubidoc.yaml"
76-
77-
def read(workingDir: File): Either[Error, Configuration] = Utils.parseFileWith(workingDir / configFile)(parse)
44+
final case class TableConfig(name: String, termName: Option[String], defName: Option[String], rows: List[BaseEntity])
7845

79-
private[atedeg] def parse(raw: String): Either[Error, Configuration] =
80-
parser.parse(raw).leftMap[Error](CirceParsingFailure).flatMap(parseJson)
8146

82-
private def parseJson(json: Json): Either[CirceDecodingFailure, Configuration] = {
83-
import io.circe.generic.auto._
84-
implicit val decodeEntity: Decoder[BaseEntity] = List(Class, Trait, Enum, Case, Type, Def)
85-
.map(t => (t.toString, BaseEntity(t, _)))
86-
.map(entry => Decoder.forProduct1(entry._1)(entry._2))
87-
.reduceLeft(_ or _)
88-
json.as[Configuration].leftMap(CirceDecodingFailure)
89-
}
90-
}
91-
92-
private object Utils {
93-
94-
private def openFile(file: File): Either[Error, String] =
95-
Try(file.contentAsString).toEither.leftMap(ExternalError)
96-
97-
def parseFileWith[A](file: File)(parser: String => Either[Error, A]): Either[Error, A] =
98-
openFile(file).flatMap(parser)
99-
}

‎src/main/scala/dev/atedeg/ConfigurationUtils.scala

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package dev.atedeg
2+
3+
import better.files.File
4+
import io.circe.{Decoder, Json}
5+
import io.circe.yaml.parser
6+
import cats.syntax.all._
7+
8+
object ConfigurationParsing {
9+
private val configFile = ".ubidoc.yaml"
10+
11+
def readConfiguration(workingDir: File): Either[Error, Configuration] = Utils.parseFileWith(workingDir / configFile)(parse)
12+
13+
private[atedeg] def parse(raw: String): Either[Error, Configuration] =
14+
parser.parse(raw).leftMap[Error](CirceParsingFailure).flatMap(parseJson)
15+
16+
private def parseJson(json: Json): Either[CirceDecodingFailure, Configuration] = {
17+
import io.circe.generic.auto._
18+
implicit val decodeEntity: Decoder[BaseEntity] = List(Class, Trait, Enum, Case, Type, Def)
19+
.map(t => (t.toString, BaseEntity(t, _)))
20+
.map(entry => Decoder.forProduct1(entry._1)(entry._2))
21+
.reduceLeft(_ or _)
22+
json.as[Configuration].leftMap(CirceDecodingFailure)
23+
}
24+
}
25+
26+
object ConfigurationValidation {
27+
def toTable(tableConfig: TableConfig, allEntities: Set[Entity]): Either[Error, Table[Entity]] = for {
28+
entities <- lookupEntities(tableConfig, allEntities)
29+
termName = tableConfig.termName.getOrElse("Term")
30+
definitionName = tableConfig.defName.getOrElse("Definition")
31+
} yield Table(tableConfig.name, termName, definitionName, entities)
32+
33+
private def lookupEntities(config: TableConfig, allEntities: Set[Entity]): Either[Error, List[Entity]] = {
34+
def lookup(baseEntity: BaseEntity): Either[Error, Entity] =
35+
allEntities.find(_.toBaseEntity == baseEntity).toRight(EntityNotFound(baseEntity))
36+
config.rows.traverse(lookup)
37+
}
38+
}

‎src/main/scala/dev/atedeg/EntityUtils.scala

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package dev.atedeg
2+
3+
import better.files.File
4+
import io.circe.{Decoder, Json}
5+
import cats.syntax.all._
6+
import io.circe.parser.{parse => parseJsonString}
7+
import Extensions._
8+
import dev.atedeg.HtmlParsing.extractTermAndDefinition
9+
10+
object EntityParsing {
11+
private val entitiesFileName = "searchData.js"
12+
13+
private def allEntitiesFile(workingDir: File): File =
14+
workingDir / "target" / "site" / "scripts" / entitiesFileName
15+
16+
def readAllEntities(workingDir: File): Either[Error, Set[Entity]] = Utils.parseFileWith(allEntitiesFile(workingDir))(parse)
17+
18+
private[atedeg] def parse(raw: String): Either[Error, Set[Entity]] =
19+
parseJsonString(raw).leftMap(CirceParsingFailure).flatMap(parseJson)
20+
21+
private def parseJson(json: Json): Either[Error, Set[Entity]] = {
22+
def build(maybeType: Option[EntityType], link: String, name: String, packageName: String): Option[Entity] =
23+
maybeType.map(Entity(_, link, name, packageName))
24+
25+
implicit val decodeEntityType: Decoder[Option[EntityType]] = Decoder.decodeString.map(EntityType.fromString)
26+
implicit val decodeEntity: Decoder[Option[Entity]] = Decoder.forProduct4("k", "l", "n", "d")(build)
27+
json.as[Set[Option[Entity]]].map(_.dropNone).leftMap(CirceDecodingFailure)
28+
}
29+
}
30+
31+
object EntityConversion {
32+
def entityToRow(entity: Entity, baseDir: File, allEntities: Set[Entity]): Either[Error, Row] =
33+
extractTermAndDefinition(baseDir / entity.sanitizedLink, entity, allEntities).map(Row(_))
34+
}

‎src/main/scala/dev/atedeg/Errors.scala

+7-2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ final case class UbidocException(error: Error) extends Exception {
99

1010
sealed trait Error
1111

12+
final case class EntityNotFound(baseEntity: BaseEntity) extends Error {
13+
override def toString: String = s"Could not find entity '$baseEntity'"
14+
}
15+
16+
1217
final case class FileNotFound(lookupDir: File, path: String) extends Error {
1318
override def toString: String = s"Could not find file '$path' in directory '${lookupDir.pathAsString}'"
1419
}
@@ -25,13 +30,13 @@ final case class AmbiguousName(name: String) extends Error {
2530
s"More than one entity with the same name: '$name'"
2631
}
2732

28-
final case class OverlappingIgnoredAndConsidered(overlapping: Set[IgnoredSelector]) extends Error {
33+
final case class OverlappingIgnoredAndConsidered(overlapping: Set[BaseEntity]) extends Error {
2934

3035
override def toString: String =
3136
s"One of the tables specified one or more entities that also appear in the ignored list: $overlapping"
3237
}
3338

34-
final case class LeftoverEntities(leftovers: Set[IgnoredSelector]) extends Error {
39+
final case class LeftoverEntities(leftovers: Set[BaseEntity]) extends Error {
3540

3641
override def toString: String =
3742
s"There are one or more entities that are not considered nor ignored, maybe you forgot about those: $leftovers"

‎src/main/scala/dev/atedeg/HtmlParsing.scala

+11-24
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,22 @@ package dev.atedeg
22

33
import better.files.File
44
import net.ruippeixotog.scalascraper.browser.{ Browser, JsoupBrowser }
5-
import net.ruippeixotog.scalascraper.scraper.ContentExtractors.*
6-
import net.ruippeixotog.scalascraper.model.*
7-
import net.ruippeixotog.scalascraper.dsl.DSL.*
8-
import cats.syntax.all.*
5+
import net.ruippeixotog.scalascraper.scraper.ContentExtractors._
6+
import net.ruippeixotog.scalascraper.model._
7+
import net.ruippeixotog.scalascraper.dsl.DSL._
8+
import cats.syntax.all._
99

1010
object HtmlParsing {
1111

12-
def extractClassLike(file: File): Either[Error, (String, String)] = for {
12+
def extractTermAndDefinition(file: File, entity: Entity, allEntities: Set[Entity]): Either[Error, (String, String)] = for {
1313
document <- JsoupBrowser().parseFile(file.toJava).asRight
14-
term <- extractTagFromDocument(file, document, "title")
15-
definition <- extractTagFromDocument(file, document, "div.doc > p")
16-
} yield (term, definition)
14+
doc <- extractDoc(file, document, entity)
15+
} yield (entity.name, doc)
1716

18-
private def extractTagFromElem(file: File, elem: Element, tag: String): Either[Error, String] =
19-
elem.tryExtract(element(tag)).map(_.childNodes).map(toMarkdown).toRight(ParseError(file, tag))
17+
def extractDoc(file: File, document: Browser#DocumentType, entity: Entity): Either[Error, String] = {
18+
val searchQuery = s"#${entity.entityId.map(_ + " > ").getOrElse("")}div.cover > div.doc"
19+
extractTagFromDocument(file, document, searchQuery)
20+
}
2021

2122
private def extractTagFromDocument(file: File, doc: Browser#DocumentType, tag: String): Either[Error, String] =
2223
doc.tryExtract(element(tag)).map(_.childNodes).map(toMarkdown).toRight(ParseError(file, tag))
@@ -34,18 +35,4 @@ object HtmlParsing {
3435
}
3536
}
3637
}
37-
38-
private def extractMany(file: File, doc: Browser#DocumentType, tag: String): Either[Error, List[Element]] =
39-
doc.tryExtract(elementList(tag)).toRight(ParseError(file, tag))
40-
41-
def extractNonClassLike(file: File, name: String): Either[Error, (String, String)] = for {
42-
document <- JsoupBrowser().parseFile(file.toJava).asRight
43-
elements <- extractMany(file, document, "div.documentableElement")
44-
definition <- elements
45-
.find(hasName(_, name))
46-
.toRight(ParseError(file, "a.documentableName"))
47-
.flatMap(extractTagFromElem(file, _, "div.cover > div.doc"))
48-
} yield (name, definition)
49-
50-
private def hasName(elem: Element, name: String): Boolean = elem.tryExtract(text("a.documentableName")).contains(name)
5138
}

‎src/main/scala/dev/atedeg/Table.scala

+9-45
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,15 @@
11
package dev.atedeg
22

3-
import java.util.Locale.UK
3+
import java.util.Locale.US
44

55
import scala.util.Try
6-
7-
import dev.atedeg.HtmlParsing.{ extractClassLike, extractNonClassLike }
8-
96
import better.files.File
10-
import cats.implicits.*
117
import net.steppschuh.markdowngenerator.table.Table.ALIGN_LEFT
128
import net.steppschuh.markdowngenerator.table.Table.Builder
139

14-
import Extensions.*
15-
16-
final case class Row(term: String, definition: String)
17-
object Row {
18-
private def normalizeName(name: String): String =
19-
name.split("(?=\\p{Upper})").toList.map(_.toLowerCase(UK)).mkString(" ").capitalize
20-
21-
def buildRow(termDefinition: (String, String)): Row =
22-
Row(normalizeName(termDefinition._1), termDefinition._2)
23-
}
10+
final case class Table[E](title: String, termName: String, definitionName: String, rows: List[E]) {
2411

25-
final case class Table(title: String, termName: String, definitionName: String, rows: List[Row]) {
12+
def map[A](f: E => A): Table[A] = Table(title, termName, definitionName, rows.map(f))
2613

2714
override def toString: String = {
2815
val builder = new Builder().withAlignment(ALIGN_LEFT).addRow(termName, definitionName)
@@ -36,36 +23,13 @@ final case class Table(title: String, termName: String, definitionName: String,
3623
}
3724
}
3825

39-
object Table {
26+
final case class Row private (term: String, definition: String)
4027

41-
def parse(config: TableConfig, lookupDir: File): Either[Error, Table] = for {
42-
rows <- config.rows.traverse(Internals.parseRow(_, lookupDir))
43-
title = config.name
44-
termName = config.termName.getOrElse("Term")
45-
definitionName = config.definitionName.getOrElse("Definition")
46-
} yield Table(title, termName, definitionName, rows)
47-
48-
private object Internals {
49-
50-
def parseRow(row: Selector, lookupDir: File): Either[Error, Row] = row match {
51-
case Class(name) => parseClassLike(name, lookupDir)
52-
case Trait(name) => parseClassLike(name, lookupDir)
53-
case Enum(name) => parseClassLike(name, lookupDir)
54-
case Type(name, lookupFile) => parseNonClassLike(name, lookupFile, lookupDir)
55-
case EnumCase(name, lookupFile) => parseNonClassLike(name, lookupFile, lookupDir)
56-
}
57-
58-
def parseClassLike(name: String, lookupDir: File): Either[Error, Row] =
59-
findFile(name + ".html", lookupDir).flatMap(extractClassLike).map(Row.buildRow)
28+
object Row {
6029

61-
def parseNonClassLike(name: String, lookupFile: String, lookupDir: File): Either[Error, Row] =
62-
findFile(lookupFile, lookupDir).flatMap(extractNonClassLike(_, name)).map(Row.buildRow)
30+
private def normalizeName(name: String): String =
31+
name.split("(?=\\p{Upper})").toList.map(_.toLowerCase(US)).mkString(" ").capitalize
6332

64-
def findFile(name: String, lookupDir: File): Either[Error, File] =
65-
lookupDir.listHtmlFiles.filter(_.name == name).toList match {
66-
case Nil => FileNotFound(lookupDir, name).asLeft
67-
case List(f) => f.asRight
68-
case _ => AmbiguousName(name).asLeft
69-
}
70-
}
33+
def apply(termDefinition: (String, String)): Row = Row(normalizeName(termDefinition._1), termDefinition._2)
34+
def apply(term: String, definition: String): Row = Row((normalizeName(term), definition))
7135
}

‎src/main/scala/dev/atedeg/TableUtils.scala

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package dev.atedeg
2+
3+
import better.files.File
4+
import cats.syntax.all._
5+
import dev.atedeg.EntityConversion.entityToRow
6+
7+
object TableUtils {
8+
9+
def entitiesToRows(table: Table[Entity], baseDir: File, allEntities: Set[Entity]): Either[Error, Table[Row]] =
10+
table.rows
11+
.traverse(entityToRow(_, baseDir, allEntities))
12+
.map(Table(table.title, table.termName, table.definitionName, _))
13+
}

‎src/main/scala/dev/atedeg/Ubidoc.scala

+20-18
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package dev.atedeg
22

3-
import java.io.{ File => JFile }
4-
5-
import better.files.{ File, FileExtensions }
6-
import cats.implicits.*
3+
import java.io.{File => JFile}
4+
import better.files.{File, FileExtensions}
5+
import ConfigurationValidation._
6+
import cats.implicits._
7+
import dev.atedeg.ConfigurationParsing.readConfiguration
8+
import dev.atedeg.EntityParsing.readAllEntities
9+
import dev.atedeg.TableUtils.entitiesToRows
710

811
object Ubidoc {
912

@@ -14,27 +17,26 @@ object Ubidoc {
1417

1518
def ubiquitousScaladocTask(lookupDir: File, targetDir: File, workingDir: File): Unit = {
1619
val result = for {
17-
config <- Configuration.read(workingDir)
18-
allEntities <- AllEntities.read(workingDir)
19-
consideredEntities = config.tables.flatMap(_.rows).toSet
20-
_ <- checkConsistency(allEntities, consideredEntities, config.ignored)
21-
tables <- config.tables.traverse(Table.parse(_, lookupDir))
22-
_ = tables.foreach(_.serialize(targetDir))
23-
} yield ()
20+
config <- readConfiguration(workingDir)
21+
allEntities <- readAllEntities(workingDir)
22+
tables <- config.tables.traverse(toTable(_, allEntities))
23+
consideredEntities = tables.flatMap(_.rows).toSet
24+
_ <- checkConsistency(allEntities.map(_.toBaseEntity), consideredEntities.map(_.toBaseEntity), config.ignored)
25+
tables <- tables.traverse(entitiesToRows(_, lookupDir, allEntities))
26+
} yield tables.foreach(_.serialize(targetDir))
2427
result match {
2528
case Left(err) => throw UbidocException(err)
26-
case Right(()) => ()
29+
case Right(()) => println("Tables generated!")
2730
}
2831
}
2932

3033
private def checkConsistency(
31-
allEntities: Set[IgnoredSelector],
32-
considered: Set[Selector],
33-
ignored: Set[IgnoredSelector],
34+
allEntities: Set[BaseEntity],
35+
considered: Set[BaseEntity],
36+
ignored: Set[BaseEntity],
3437
): Either[Error, Unit] = {
35-
val c = considered.map(_.toIgnored)
36-
val consideredAndIgnoredIntersection = c.intersect(ignored)
37-
val leftoverEntities = allEntities.diff(c).diff(ignored)
38+
val consideredAndIgnoredIntersection = considered.intersect(ignored)
39+
val leftoverEntities = allEntities.diff(considered).diff(ignored)
3840
if (consideredAndIgnoredIntersection.nonEmpty)
3941
OverlappingIgnoredAndConsidered(consideredAndIgnoredIntersection).asLeft
4042
else if (leftoverEntities.nonEmpty) LeftoverEntities(leftoverEntities).asLeft

0 commit comments

Comments
 (0)