mirrors/notebook
3
0
Fork 0
mirror of https://github.com/indentlabs/notebook.git synced 2025-10-26 11:19:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1124 from indentlabs/mar28

Browse Source 
Mar 28 changes
This commit is contained in:
Andrew Brown 2022-03-29 02:28:35 -05:00 committed by GitHub
commit ffe3b45baf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 374 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/documents_controller.rb
View File

@ -4,8 +4,6 @@ class DocumentsController < ApplicationController
# todo Uh, this is a hack. The CSRF token on document editor model to add entities is being rejected... for whatever reason.
skip_before_action :verify_authenticity_token, only: [:link_entity]
skip_before_action :cache_most_used_page_information, only: [:update]
before_action :set_document, only: [:show, :analysis, :plaintext, :queue_analysis, :edit, :destroy]
before_action :set_sidenav_expansion, except: [:plaintext]
before_action :set_navbar_color, except: [:plaintext]

13
app/controllers/export_controller.rb
View File

@ -2,6 +2,9 @@ class ExportController < ApplicationController
before_action :authenticate_user!
before_action :whitelist_pluralized_model, only: [:csv]
skip_before_action :cache_most_used_page_information, only: [:outline, :notebook_json]
skip_before_action :cache_forums_unread_counts, only: [:outline, :notebook_json]
def index
@sidenav_expansion = 'my account'
end
@ -11,11 +14,17 @@ class ExportController < ApplicationController
end
def outline
send_data content_to_outline, filename: "notebook-#{Date.today}.txt"
export = ExportService.text_outline_export(current_user.universes.pluck(:id))
send_data export, filename: "notebook-#{Date.today}.txt"
end
def markdown
export = ExportService.text_markdown_export(current_user.universes.pluck(:id))
send_data export, filename: "notebook-#{Date.today}.md"
end
def notebook_json
json_dump = current_user.content.except('Document').except('Timeline').map { |category, content| {"#{category}": fill_relations(category.constantize, content)} }.to_json
json_dump = ExportService.json_export(current_user.universes.pluck(:id))
send_data json_dump, filename: "notebook-#{Date.today}.json"
end

14
app/jobs/export_job.rb Normal file
View File

@ -0,0 +1,14 @@
class ExportJob < ApplicationJob
queue_as :export
def perform(*args)
universe_ids = args.shift
format = args.shift
# design: left side 1/2 - list universes with toggle/checkboxes to include
# right side 1/2 - radios for format, checkboxes for options after
# (include hidden? include IDs? include blank fields?)
end
end

8
app/models/concerns/has_page_references.rb
View File

@ -6,8 +6,9 @@ module HasPageReferences
included do
# Pages that reference this one
has_many :incoming_page_references,
as: :referenced_page,
class_name: PageReference.name
as: :referenced_page,
class_name: PageReference.name,
dependent: :destroy
Rails.application.config.content_type_names[:all].each do |page_type|
has_many "referencing_#{page_type.downcase}_pages".to_sym,
through: :incoming_page_references,
@ -26,7 +27,8 @@ module HasPageReferences
# Pages referenced by this one
has_many :outgoing_page_references,
as: :referencing_page,
class_name: PageReference.name
class_name: PageReference.name,
dependent: :destroy
Rails.application.config.content_type_names[:all].each do |page_type|
has_many "referenced_#{page_type.downcase}_pages".to_sym,
through: :outgoing_page_references,

2
app/models/page_data/attribute_field.rb
View File

@ -7,6 +7,8 @@ class AttributeField < ApplicationRecord
belongs_to :attribute_category
has_many :attribute_values, class_name: 'Attribute', dependent: :destroy
has_many :page_references, dependent: :destroy
validates_presence_of :user_id
acts_as_list scope: [:user_id, :attribute_category_id]

323
app/services/export_service.rb Normal file
View File

@ -0,0 +1,323 @@
class ExportService < Service
INCLUDE_EMPTY_FIELD_LABELS = false
def self.text_outline_export(universe_ids)
page_types = Rails.application.config.content_types[:all]
temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id
export_text = StringIO.new
categories = fetch_categories(user_id: temporary_user_id_reference)
fields = fetch_fields(category_ids: categories.map(&:id))
values = fetch_values(field_ids: fields.map(&:id))
page_types.each do |page_type|
# Get all pages in the given universe(s)
pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids)
: page_type.where(universe_id: universe_ids)
if pages.any?
export_text << "# #{page_type.name.pluralize}\n"
categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase }
fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id }
pages.each do |page|
export_text << "* Name: #{page.name}\n"
export_text << " ID: #{page.id}\n"
values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id }
categories_for_this_page_type.each do |category|
# Delay printing our category header until we've found a value to print, so we don't print
# empty headers and blank sections.
printed_category_header = false
fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id }
values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id }
fields_for_this_category.each do |field|
value_for_this_field = values_for_these_fields.detect do |value|
value.attribute_field_id == field.id &&
value.entity_type == page_type.name &&
value.entity_id == page.id
end
# If we have a field to print but haven't printed the header yet, print it
if value_for_this_field.try(:value).try(:present?) && !printed_category_header
export_text << " #{category.label}\n"
printed_category_header = true
end
case field.field_type
when 'text_area', 'textarea'
# For answers to text fields, we can just output their value directly
if value_for_this_field.present? || INCLUDE_EMPTY_FIELD_LABELS
export_text << " #{field.label}: #{value_for_this_field.try :value}\n"
end
when 'link'
# For link fields, we have link codes ([[Character-1]]) and need to translate
# them into the linked page's name in order for them to actually be useful
if value_for_this_field.present?
json_list = JSON.parse(value_for_this_field.value)
formatted_names = json_list.map do |link_code|
query_link_code_with_cache(*link_code.split('-')).name
end
export_text << " #{field.label}: #{formatted_names.to_sentence}\n"
else
if INCLUDE_EMPTY_FIELD_LABELS
export_text << " #{field.label}:\n"
end
end
end
end
end
export_text << "\n" # spacer between pages
end
export_text << "\n" # spacer between sections
end
end
return export_text.string
end
def self.text_markdown_export(universe_ids)
page_types = Rails.application.config.content_types[:all]
temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id
export_text = StringIO.new
categories = fetch_categories(user_id: temporary_user_id_reference)
fields = fetch_fields(category_ids: categories.map(&:id))
values = fetch_values(field_ids: fields.map(&:id))
page_types.each do |page_type|
# Get all pages in the given universe(s)
pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids)
: page_type.where(universe_id: universe_ids)
if pages.any?
export_text << "# #{page_type.name.pluralize}\n"
categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase }
fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id }
pages.each do |page|
export_text << "## #{page.name}\n"
export_text << "ID: #{page.id}\n"
values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id }
categories_for_this_page_type.each do |category|
# Delay printing our category header until we've found a value to print, so we don't print
# empty headers and blank sections.
printed_category_header = false
fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id }
values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id }
fields_for_this_category.each do |field|
value_for_this_field = values_for_these_fields.detect do |value|
value.attribute_field_id == field.id &&
value.entity_type == page_type.name &&
value.entity_id == page.id
end
# If we have a field to print but haven't printed the header yet, print it
if value_for_this_field.try(:value).try(:present?) && !printed_category_header
export_text << "### #{category.label}\n"
printed_category_header = true
end
case field.field_type
when 'text_area', 'textarea'
# For answers to text fields, we can just output their value directly
if value_for_this_field.try(:value).present? || INCLUDE_EMPTY_FIELD_LABELS
export_text << "#### #{field.label}\n#{value_for_this_field.try :value}\n"
end
when 'link'
# For link fields, we have link codes ([[Character-1]]) and need to translate
# them into the linked page's name in order for them to actually be useful
if value_for_this_field.present?
json_list = JSON.parse(value_for_this_field.value)
formatted_names = json_list.map do |link_code|
link_type, link_id = link_code.split('-')
page_ref = query_link_code_with_cache(link_type, link_id)
"[#{page_ref.name}](https://www.notebook.ai/plan/#{link_type.downcase.pluralize}/#{link_id})"
end
export_text << "#### #{field.label}\n#{formatted_names.to_sentence}\n"
else
if INCLUDE_EMPTY_FIELD_LABELS
export_text << "#### #{field.label}\n"
end
end
end
end
end
export_text << "\n" # spacer between pages
end
export_text << "\n" # spacer between sections
end
end
return export_text.string
end
def self.csv_export(universe_ids, separator=',')
# todo: figure out zip
end
def self.json_export(universe_ids)
page_types = Rails.application.config.content_types[:all]
temporary_user_id_reference = Universe.find_by(id: universe_ids).user_id
export_object = {}
categories = fetch_categories(user_id: temporary_user_id_reference)
fields = fetch_fields(category_ids: categories.map(&:id))
values = fetch_values(field_ids: fields.map(&:id))
page_types.each do |page_type|
# Get all pages in the given universe(s)
pages = (page_type.name == 'Universe') ? page_type.where(id: universe_ids)
: page_type.where(universe_id: universe_ids)
if pages.any?
export_object[page_type.name] = []
categories_for_this_page_type = categories.select { |c| c.entity_type == page_type.name.downcase }
fields_for_this_page_type = fields.select { |f| categories_for_this_page_type.map(&:id).include? f.attribute_category_id }
page_object = {}
pages.each do |page|
page_object['name'] = page.name
page_object['id'] = page.id
values_for_this_page = values.select { |v| v.entity_type == page_type.name && v.entity_id == page.id }
categories_for_this_page_type.each do |category|
page_object[category.label] ||= {}
fields_for_this_category = fields_for_this_page_type.select { |f| f.attribute_category_id == category.id }
values_for_these_fields = values_for_this_page.select { |v| fields_for_this_category.map(&:id).include? v.attribute_field_id }
fields_for_this_category.each do |field|
value_for_this_field = values_for_these_fields.detect do |value|
value.attribute_field_id == field.id &&
value.entity_type == page_type.name &&
value.entity_id == page.id
end
case field.field_type
when 'text_area', 'textarea'
# For answers to text fields, we can just output their value directly
if value_for_this_field.try(:value).present? || INCLUDE_EMPTY_FIELD_LABELS
page_object[category.label][field.label] = value_for_this_field.try :value
end
when 'link'
# For link fields, we have link codes ([[Character-1]]) and need to translate
# them into the linked page's name in order for them to actually be useful
if value_for_this_field.present?
json_list = JSON.parse(value_for_this_field.value)
formatted_name_objects = json_list.map do |link_code|
link_type, link_id = link_code.split('-')
page_ref = query_link_code_with_cache(link_type, link_id)
name_object = {
name: page_ref.name,
id: page_ref.id,
}
end
page_object[category.label][field.label] = formatted_name_objects
else
if INCLUDE_EMPTY_FIELD_LABELS
page_object[category.label][field.label] = nil
end
end
end
end
end
end
# Append the JSON page object we just built to the list of pages of that kind
export_object[page_type.name].push page_object
end
end
return export_object.to_json
end
def self.xml_export(universe_ids)
end
def self.yaml_export(universe_ids)
end
def self.html_export(universe_ids)
end
def self.scrivener_export(universe_ids)
end
private
def self.fetch_categories(user_id:)
AttributeCategory.where(user_id: user_id)
.order('position ASC')
.select(:id, :label, :entity_type)
end
def self.fetch_fields(category_ids:)
AttributeField.where(attribute_category_id: category_ids)
.order('position ASC')
.select(:id, :label, :attribute_category_id, :field_type)
end
def self.fetch_values(field_ids:)
Attribute.where(attribute_field_id: field_ids)
.select(:attribute_field_id, :value, :entity_type, :entity_id)
end
def self.fetch_universes(universe_ids)
end
def self.query_link_code_with_cache(content_type_name, content_id)
cache_key = "#{content_type_name}-#{content_id}"
# Pull from the cache to avoid a query if we can
@content_cache ||= {}
if @content_cache.key?(cache_key)
return @content_cache[cache_key]
end
# TODO: we should probably whitelist content_type from valid page types here
# If there's no cache, we unfortunately need to do a query to resolve the link code
content = class_from_name(content_type_name).find(content_id)
@content_cache[cache_key] = content
return content
end
def self.class_from_name(content_type_name)
# This lookup is ~3x faster than .constantize
Rails.application.config.content_types_by_name[content_type_name]
end
end

1
config/routes.rb
View File

@ -155,6 +155,7 @@ Rails.application.routes.draw do
get '/', to: 'export#index', as: :notebook_export
get '/outline', to: 'export#outline', as: :notebook_outline
get '/markdown', to: 'export#markdown', as: :notebook_markdown
get '/notebook.json', to: 'export#notebook_json', as: :notebook_json
get '/notebook.xml', to: 'export#notebook_xml', as: :notebook_xml
get '/notebook.yml', to: 'export#notebook_yml', as: :notebook_yml

1
config/sidekiq.yml
View File

@ -6,4 +6,5 @@
- default
- cache
- notifications
- export
- low_priority

17
lib/tasks/data_integrity.rake
View File

@ -112,6 +112,23 @@ namespace :data_integrity do
end
desc "Remove orphan page references"
task remove_orphan_page_references: :environment do
PageReference.find_each do |reference|
if reference.referencing_page.nil?
puts "Deleting reference #{reference.id}"
reference.destroy
next
end
if reference.referenced_page.nil?
puts "Deleting reference #{reference.id}"
reference.destroy
next
end
end
end
desc "Ensure all users have the correct upload bandwidth amounts"
task correct_bandwidths: :environment do
base_bandwidth = User.new.upload_bandwidth_kb # 50_000